diff --git a/firestore-bigquery-export/scripts/import/package-lock.json b/firestore-bigquery-export/scripts/import/package-lock.json index af1f92d6c..f48d23d66 100644 --- a/firestore-bigquery-export/scripts/import/package-lock.json +++ b/firestore-bigquery-export/scripts/import/package-lock.json @@ -1,12 +1,12 @@ { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.26", + "version": "0.1.27", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.26", + "version": "0.1.27", "license": "Apache-2.0", "dependencies": { "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.40", @@ -505,30 +505,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, "node_modules/@fastify/busboy": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.1.1.tgz", @@ -1810,34 +1786,6 @@ "node": ">= 10" } }, - "node_modules/@tsconfig/node10": { - "version": "1.0.9", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.3", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/@types/babel__core": { "version": "7.20.5", "dev": true, @@ -2098,29 +2046,6 @@ "node": ">= 0.6" } }, - "node_modules/acorn": { - "version": "8.11.3", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.2.0", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/agent-base": { "version": "6.0.2", "license": "MIT", @@ -2170,13 +2095,6 @@ "node": ">= 8" } }, - "node_modules/arg": { - "version": "4.1.3", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/argparse": { "version": "1.0.10", "dev": true, @@ -2922,13 +2840,6 @@ "node": ">=8" } }, - "node_modules/create-require": { - "version": "1.1.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/cross-spawn": { "version": "7.0.6", "dev": true, @@ -3108,16 +3019,6 @@ "node": ">=8" } }, - "node_modules/diff": { - "version": "4.0.2", - "dev": true, - "license": "BSD-3-Clause", - "optional": true, - "peer": true, - "engines": { - "node": ">=0.3.1" - } - }, "node_modules/diff-sequences": { "version": "29.6.3", "dev": true, @@ -7929,50 +7830,6 @@ "node": ">=10" } }, - "node_modules/ts-node": { - "version": "10.9.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -8210,13 +8067,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true - }, "node_modules/v8-to-istanbul": { "version": "9.2.0", "dev": true, @@ -8438,16 +8288,6 @@ "node": ">=12" } }, - "node_modules/yn": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=6" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "devOptional": true, diff --git a/firestore-bigquery-export/scripts/import/package.json b/firestore-bigquery-export/scripts/import/package.json index 989b5b049..7955018fb 100644 --- a/firestore-bigquery-export/scripts/import/package.json +++ b/firestore-bigquery-export/scripts/import/package.json @@ -1,6 +1,6 @@ { "name": "@firebaseextensions/fs-bq-import-collection", - "version": "0.1.26", + "version": "0.1.27", "description": "Import a Firestore Collection into a BigQuery Changelog Table", "main": "./lib/index.js", "repository": { diff --git a/firestore-bigquery-export/scripts/import/src/config.ts b/firestore-bigquery-export/scripts/import/src/config.ts index ce785f3a8..904250097 100644 --- a/firestore-bigquery-export/scripts/import/src/config.ts +++ b/firestore-bigquery-export/scripts/import/src/config.ts @@ -84,6 +84,13 @@ const questions = [ PROJECT_ID_MAX_CHARS ), }, + { + message: + "What is your Firestore database instance ID? (Use '(default)' for the default database)", + name: "firestoreInstanceId", + type: "input", + default: "(default)", + }, { message: "What is your BigQuery project ID?", name: "bigQueryProject", @@ -271,10 +278,12 @@ export async function parseConfig(): Promise { cursorPositionFile, failedBatchOutput: program.failedBatchOutput, transformFunctionUrl: program.transformFunctionUrl, + firestoreInstanceId: program.firestoreInstanceId || "(default)", }; } const { project, + firestoreInstanceId, sourceCollectionPath, bigQueryProject, dataset, @@ -314,6 +323,7 @@ export async function parseConfig(): Promise { cursorPositionFile, failedBatchOutput, transformFunctionUrl, + firestoreInstanceId: firestoreInstanceId || "(default)", }; } diff --git a/firestore-bigquery-export/scripts/import/src/index.ts b/firestore-bigquery-export/scripts/import/src/index.ts index 9a6409134..b3c11e4a4 100644 --- a/firestore-bigquery-export/scripts/import/src/index.ts +++ b/firestore-bigquery-export/scripts/import/src/index.ts @@ -17,6 +17,7 @@ */ import { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker"; import * as firebase from "firebase-admin"; +import { getFirestore } from "firebase-admin/firestore"; import * as fs from "fs"; import * as util from "util"; @@ -50,6 +51,7 @@ const run = async (): Promise => { useEmulator, cursorPositionFile, transformFunctionUrl, + firestoreInstanceId, } = config; if (useEmulator) { console.log("Using emulator"); @@ -63,14 +65,23 @@ const run = async (): Promise => { // Initialize Firebase // This uses applicationDefault to authenticate // Please see https://cloud.google.com/docs/authentication/production + let app: firebase.app.App; if (!firebase.apps.length) { - firebase.initializeApp({ + app = firebase.initializeApp({ projectId: projectId, credential: firebase.credential.applicationDefault(), databaseURL: `https://${projectId}.firebaseio.com`, }); + } else { + app = firebase.app(); } + // Get the Firestore instance for the specified database + const db = + firestoreInstanceId && firestoreInstanceId !== "(default)" + ? getFirestore(app, firestoreInstanceId) + : getFirestore(app); + // We pass in the application-level "tableId" here. The tracker determines // the name of the raw changelog from this field. // TODO: fix this type, it should include clustering apparently @@ -83,6 +94,7 @@ const run = async (): Promise => { useNewSnapshotQuerySyntax, bqProjectId: bigQueryProjectId, transformFunction: transformFunctionUrl, + firestoreInstanceId: firestoreInstanceId, }); await initializeDataSink(dataSink, config); @@ -103,10 +115,10 @@ const run = async (): Promise => { if (await exists(cursorPositionFile)) { let cursorDocumentId = (await read(cursorPositionFile)).toString(); - cursor = await firebase.firestore().doc(cursorDocumentId).get(); + cursor = await db.doc(cursorDocumentId).get(); logs.resumingImport(config, cursorDocumentId); } - const totalRowsImported = await runSingleThread(dataSink, config, cursor); + const totalRowsImported = await runSingleThread(dataSink, config, cursor, db); try { await unlink(cursorPositionFile); } catch (e) { diff --git a/firestore-bigquery-export/scripts/import/src/program.ts b/firestore-bigquery-export/scripts/import/src/program.ts index 87982aa16..24f4df2c8 100644 --- a/firestore-bigquery-export/scripts/import/src/program.ts +++ b/firestore-bigquery-export/scripts/import/src/program.ts @@ -65,5 +65,10 @@ export const getCLIOptions = () => { .option( "-f, --failed-batch-output ", "Path to the JSON file where failed batches will be recorded." + ) + .option( + "--firestore-instance-id ", + "The Firestore database instance ID. Use '(default)' for the default database.", + "(default)" ); }; diff --git a/firestore-bigquery-export/scripts/import/src/run-multi-thread.ts b/firestore-bigquery-export/scripts/import/src/run-multi-thread.ts index 259c563ff..fc7f0a71c 100644 --- a/firestore-bigquery-export/scripts/import/src/run-multi-thread.ts +++ b/firestore-bigquery-export/scripts/import/src/run-multi-thread.ts @@ -1,4 +1,5 @@ import * as firebase from "firebase-admin"; +import { getFirestore } from "firebase-admin/firestore"; import { cpus } from "os"; import { pool } from "workerpool"; import * as logs from "./logs"; @@ -20,19 +21,24 @@ export async function runMultiThread(config: CliConfig): Promise { PROJECT_ID: config.projectId, GOOGLE_CLOUD_PROJECT: config.projectId, GCLOUD_PROJECT: config.projectId, + FIRESTORE_INSTANCE_ID: config.firestoreInstanceId, FAILED_BATCH_OUTPUT: config.failedBatchOutput || "", ...process.env, }, }, }); - const query = firebase - .firestore() - .collectionGroup( - config.sourceCollectionPath.split("/")[ - config.sourceCollectionPath.split("/").length - 1 - ] - ); + const app = firebase.app(); + const db = + config.firestoreInstanceId && config.firestoreInstanceId !== "(default)" + ? getFirestore(app, config.firestoreInstanceId) + : getFirestore(app); + + const query = db.collectionGroup( + config.sourceCollectionPath.split("/")[ + config.sourceCollectionPath.split("/").length - 1 + ] + ); const partitionsList = query.getPartitions(config.batchSize); diff --git a/firestore-bigquery-export/scripts/import/src/run-single-thread.ts b/firestore-bigquery-export/scripts/import/src/run-single-thread.ts index 87b28e02c..9ed00ae92 100644 --- a/firestore-bigquery-export/scripts/import/src/run-single-thread.ts +++ b/firestore-bigquery-export/scripts/import/src/run-single-thread.ts @@ -3,6 +3,7 @@ import { FirestoreDocumentChangeEvent, } from "@firebaseextensions/firestore-bigquery-change-tracker"; import * as firebase from "firebase-admin"; +import { getFirestore } from "firebase-admin/firestore"; import * as fs from "fs"; import * as util from "util"; @@ -18,25 +19,36 @@ const write = util.promisify(fs.writeFile); export function getQuery( config: CliConfig, - cursor?: firebase.firestore.DocumentSnapshot + cursor?: firebase.firestore.DocumentSnapshot, + db?: firebase.firestore.Firestore ): firebase.firestore.Query { - const { sourceCollectionPath, batchSize, queryCollectionGroup } = config; + const { + sourceCollectionPath, + batchSize, + queryCollectionGroup, + firestoreInstanceId, + } = config; + + // Use provided db or get the appropriate instance + if (!db) { + const app = firebase.app(); + db = + firestoreInstanceId && firestoreInstanceId !== "(default)" + ? getFirestore(app, firestoreInstanceId) + : getFirestore(app); + } let collectionOrCollectionGroup: | firebase.firestore.CollectionGroup | firebase.firestore.CollectionReference; if (queryCollectionGroup) { - collectionOrCollectionGroup = firebase - .firestore() - .collectionGroup( - sourceCollectionPath.split("/")[ - sourceCollectionPath.split("/").length - 1 - ] - ); + collectionOrCollectionGroup = db.collectionGroup( + sourceCollectionPath.split("/")[ + sourceCollectionPath.split("/").length - 1 + ] + ); } else { - collectionOrCollectionGroup = firebase - .firestore() - .collection(sourceCollectionPath); + collectionOrCollectionGroup = db.collection(sourceCollectionPath); } let query = collectionOrCollectionGroup.limit(batchSize); @@ -47,30 +59,35 @@ export function getQuery( return query; } -async function verifyCollectionExists(config: CliConfig): Promise { - const { sourceCollectionPath, queryCollectionGroup } = config; +async function verifyCollectionExists( + config: CliConfig, + db?: firebase.firestore.Firestore +): Promise { + const { sourceCollectionPath, queryCollectionGroup, firestoreInstanceId } = + config; + + // Use provided db or get the appropriate instance + if (!db) { + const app = firebase.app(); + db = + firestoreInstanceId && firestoreInstanceId !== "(default)" + ? getFirestore(app, firestoreInstanceId) + : getFirestore(app); + } try { if (queryCollectionGroup) { const sourceCollectionPathParts = sourceCollectionPath.split("/"); const collectionName = sourceCollectionPathParts[sourceCollectionPathParts.length - 1]; - const snapshot = await firebase - .firestore() - .collectionGroup(collectionName) - .limit(1) - .get(); + const snapshot = await db.collectionGroup(collectionName).limit(1).get(); if (snapshot.empty) { throw new Error( `No documents found in collection group: ${collectionName}` ); } } else { - const snapshot = await firebase - .firestore() - .collection(sourceCollectionPath) - .limit(1) - .get(); + const snapshot = await db.collection(sourceCollectionPath).limit(1).get(); if (snapshot.empty) { throw new Error( `Collection does not exist or is empty: ${sourceCollectionPath}` @@ -91,11 +108,12 @@ export async function runSingleThread( config: CliConfig, cursor: | firebase.firestore.DocumentSnapshot - | undefined + | undefined, + db?: firebase.firestore.Firestore ) { let totalRowsImported = 0; - await verifyCollectionExists(config); + await verifyCollectionExists(config, db); await initializeFailedBatchOutput(config.failedBatchOutput); @@ -104,7 +122,7 @@ export async function runSingleThread( await write(config.cursorPositionFile, cursor.ref.path); } - let query = getQuery(config, cursor); + let query = getQuery(config, cursor, db); const snapshot = await query.get(); const docs = snapshot.docs; diff --git a/firestore-bigquery-export/scripts/import/src/types.ts b/firestore-bigquery-export/scripts/import/src/types.ts index e2dfdcd18..2c384000b 100644 --- a/firestore-bigquery-export/scripts/import/src/types.ts +++ b/firestore-bigquery-export/scripts/import/src/types.ts @@ -17,6 +17,7 @@ export interface CliConfig { cursorPositionFile: string; failedBatchOutput?: string; transformFunctionUrl?: string; + firestoreInstanceId: string; } export interface CliConfigError { diff --git a/firestore-bigquery-export/scripts/import/src/worker.ts b/firestore-bigquery-export/scripts/import/src/worker.ts index da2a8672f..f57848408 100644 --- a/firestore-bigquery-export/scripts/import/src/worker.ts +++ b/firestore-bigquery-export/scripts/import/src/worker.ts @@ -3,6 +3,7 @@ import { FirestoreDocumentChangeEvent, } from "@firebaseextensions/firestore-bigquery-change-tracker"; import * as firebase from "firebase-admin"; +import { getFirestore } from "firebase-admin/firestore"; import { worker } from "workerpool"; import { getRowsFromDocs, recordFailedBatch } from "./helper"; import { CliConfig, SerializableQuery } from "./types"; @@ -28,21 +29,30 @@ async function processDocuments( datasetLocation, batchSize, failedBatchOutput, + firestoreInstanceId, } = config; // Initialize Firebase Admin SDK if not already initialized in this worker + let app: firebase.app.App; if (!firebase.apps.length) { - firebase.initializeApp({ + app = firebase.initializeApp({ projectId: projectId, credential: firebase.credential.applicationDefault(), databaseURL: `https://${projectId}.firebaseio.com`, }); + } else { + app = firebase.app(); } + // Get the appropriate Firestore instance + const db = + firestoreInstanceId && firestoreInstanceId !== "(default)" + ? getFirestore(app, firestoreInstanceId) + : getFirestore(app); + // Construct base query for the collection group // Using collectionGroup allows querying across all collections with the same ID - let query = firebase - .firestore() + let query = db .collectionGroup(sourceCollectionPath.split("/").pop()!) .orderBy(firebase.firestore.FieldPath.documentId(), "asc"); @@ -92,7 +102,7 @@ async function processDocuments( ); } - query = query.startAt(firebase.firestore().doc(documentPath)); + query = query.startAt(db.doc(documentPath)); } if (serializableQuery.endAt?.values?.[0]?.referenceValue) { const fullPath = serializableQuery.endAt.values[0].referenceValue; @@ -111,7 +121,7 @@ async function processDocuments( ); } - query = query.endBefore(firebase.firestore().doc(documentPath)); + query = query.endBefore(db.doc(documentPath)); } if (serializableQuery.offset) { query = query.offset(serializableQuery.offset); @@ -131,6 +141,7 @@ async function processDocuments( skipInit: true, useNewSnapshotQuerySyntax: config.useNewSnapshotQuerySyntax, transformFunction: config.transformFunctionUrl, + firestoreInstanceId: firestoreInstanceId, }); // Process documents in batches until we've covered the entire partition diff --git a/storage-resize-images/CHANGELOG.md b/storage-resize-images/CHANGELOG.md index 3cd98058a..0e7d219ff 100644 --- a/storage-resize-images/CHANGELOG.md +++ b/storage-resize-images/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.3.0 + +fix! - remove backfill, due to architectural flaws. + ## Version 0.2.10 feat - added param for adjusting backfill max batch size diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index 689e080c1..69d5944a9 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -144,7 +144,7 @@ You can find more information about this extension in the following articles: * Sizes of resized images: What sizes of images would you like (in pixels)? Enter the sizes as a comma-separated list of WIDTHxHEIGHT values. Learn more about [how this parameter works](https://firebase.google.com/products/extensions/storage-resize-images). -* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone, and if you reconfigure this instance to use different image dimensions, you won't be able to backfill deleted images. +* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone. * Make resized images public: Do you want to make the resized images public automatically? So you can access them by URL. For example: https://storage.googleapis.com/{bucket}/{path} @@ -182,13 +182,6 @@ Leave this field empty if you do not want to store failed images in a separate d * Cloud Function memory: Memory of the function responsible of resizing images. Choose how much memory to give to the function that resize images. (For animated GIF => GIF we recommend using a minimum of 2GB). -* Backfill existing images: Should existing, unresized images in the Storage bucket be resized as well? - - -* Backfill batch size: The maximum number of images to resize in a single batch. By default, the function is configured to resize 3 images at a time. This is a conservative default to work with smallest memory option (512 MB). -WARNING: Ensure your function has enough memory to handle the batch size. - - * Assign new access token: Should resized images have a new access token assigned to them, different from the original image? @@ -207,8 +200,6 @@ WARNING: Ensure your function has enough memory to handle the batch size. * **generateResizedImage:** Listens for new images uploaded to your specified Cloud Storage bucket, resizes the images, then stores the resized images in the same bucket. Optionally keeps or deletes the original images. -* **backfillResizedImages:** Handles tasks from startBackfill to resize existing images. - **APIs Used**: diff --git a/storage-resize-images/extension.yaml b/storage-resize-images/extension.yaml index 1e593295e..69d3fec20 100644 --- a/storage-resize-images/extension.yaml +++ b/storage-resize-images/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: storage-resize-images -version: 0.2.10 +version: 0.3.0 specVersion: v1beta displayName: Resize Images @@ -65,14 +65,15 @@ resources: eventTrigger: eventType: google.storage.object.finalize resource: projects/_/buckets/${param:IMG_BUCKET} - - name: backfillResizedImages - type: firebaseextensions.v1beta.function - description: >- - Handles tasks from startBackfill to resize existing images. - properties: - runtime: nodejs20 - availableMemoryMb: ${param:FUNCTION_MEMORY} - taskQueueTrigger: {} + # Backfill feature disabled - commented out to preserve code + # - name: backfillResizedImages + # type: firebaseextensions.v1beta.function + # description: >- + # Handles tasks from startBackfill to resize existing images. + # properties: + # runtime: nodejs20 + # availableMemoryMb: ${param:FUNCTION_MEMORY} + # taskQueueTrigger: {} params: - param: IMG_BUCKET @@ -111,9 +112,7 @@ params: label: Deletion of original file description: >- Do you want to automatically delete the original file from the Cloud - Storage bucket? Warning: these deletions cannot be undone, and if you - reconfigure this instance to use different image dimensions, you won't be - able to backfill deleted images. + Storage bucket? Warning: these deletions cannot be undone. type: select options: - label: Don't delete @@ -316,32 +315,33 @@ params: required: true immutable: false - - param: DO_BACKFILL - label: Backfill existing images - description: > - Should existing, unresized images in the Storage bucket be resized as - well? - type: select - required: true - options: - - label: Yes - value: true - - label: No - value: false - - - param: BACKFILL_BATCH_SIZE - label: Backfill batch size - description: > - The maximum number of images to resize in a single batch. By default, the - function is configured to resize 3 images at a time. This is a - conservative default to work with smallest memory option (512 MB). - - WARNING: Ensure your function has enough memory to handle the batch size. - type: string - default: 3 - validationRegex: ^[1-9][0-9]*$ - validationErrorMessage: Please provide a valid number. - required: false + # Backfill parameters disabled - commented out to preserve code + # - param: DO_BACKFILL + # label: Backfill existing images + # description: > + # Should existing, unresized images in the Storage bucket be resized as + # well? + # type: select + # required: true + # options: + # - label: Yes + # value: true + # - label: No + # value: false + + # - param: BACKFILL_BATCH_SIZE + # label: Backfill batch size + # description: > + # The maximum number of images to resize in a single batch. By default, the + # function is configured to resize 3 images at a time. This is a + # conservative default to work with smallest memory option (512 MB). + # + # WARNING: Ensure your function has enough memory to handle the batch size. + # type: string + # default: 3 + # validationRegex: ^[1-9][0-9]*$ + # validationErrorMessage: Please provide a valid number. + # required: false - param: REGENERATE_TOKEN label: Assign new access token @@ -426,14 +426,14 @@ events: description: Occurs when the function is settled. Provides no customized data other than the context. - -lifecycleEvents: - onInstall: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} - onUpdate: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} - onConfigure: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# Lifecycle events disabled - backfill feature commented out +# lifecycleEvents: +# onInstall: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# onUpdate: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# onConfigure: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} diff --git a/storage-resize-images/functions/__tests__/__mocks__/src/config.ts b/storage-resize-images/functions/__tests__/__mocks__/src/config.ts index 5e81e04b8..c98e07a3c 100644 --- a/storage-resize-images/functions/__tests__/__mocks__/src/config.ts +++ b/storage-resize-images/functions/__tests__/__mocks__/src/config.ts @@ -51,7 +51,7 @@ export const config = { bucket: "extensions-testing.appspot.com", cacheControlHeader: undefined, - doBackfill: false, + // doBackfill: false, imageSizes: ["200x200"], regenerateToken: false, makePublic: false, diff --git a/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap b/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap index 64999bd9e..20f63350a 100644 --- a/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap @@ -3,13 +3,11 @@ exports[`extension configuration detected from environment variables 1`] = ` Object { "animated": false, - "backfillBatchSize": 3, "bucket": "extensions-testing.appspot.com", "cacheControlHeader": undefined, "contentFilterLevel": null, "customFilterPrompt": null, "deleteOriginalFile": 0, - "doBackfill": false, "excludePathList": undefined, "failedImagesPath": undefined, "imageSizes": Array [ diff --git a/storage-resize-images/functions/__tests__/filters.test.ts b/storage-resize-images/functions/__tests__/filters.test.ts index 8db9b27f0..401a95467 100644 --- a/storage-resize-images/functions/__tests__/filters.test.ts +++ b/storage-resize-images/functions/__tests__/filters.test.ts @@ -11,7 +11,7 @@ jest.mock("../src/config", () => { imgSizes: ["200x200"], resizedImagesPath: undefined, deleteOriginalFile: "true", - backfillBatchSize: 3, + // backfillBatchSize: 3, }, }; }); diff --git a/storage-resize-images/functions/__tests__/function.test.ts b/storage-resize-images/functions/__tests__/function.test.ts index 7111f9284..8d869c5bd 100644 --- a/storage-resize-images/functions/__tests__/function.test.ts +++ b/storage-resize-images/functions/__tests__/function.test.ts @@ -27,7 +27,7 @@ jest.mock("../src/config", () => { imgSizes: ["200x200"], resizedImagesPath: undefined, deleteOriginalFile: "true", - backfillBatchSize: 3, + // backfillBatchSize: 3, }, }; }); diff --git a/storage-resize-images/functions/src/config.ts b/storage-resize-images/functions/src/config.ts index e5436fd90..863ae77b7 100644 --- a/storage-resize-images/functions/src/config.ts +++ b/storage-resize-images/functions/src/config.ts @@ -68,7 +68,8 @@ export const convertHarmBlockThreshold: ( export const config = { bucket: process.env.IMG_BUCKET, cacheControlHeader: process.env.CACHE_CONTROL_HEADER, - doBackfill: process.env.DO_BACKFILL === "true", + // Backfill feature disabled - commented out to preserve code + // doBackfill: process.env.DO_BACKFILL === "true", imageSizes: process.env.IMG_SIZES.split(","), regenerateToken: process.env.REGENERATE_TOKEN == "true", makePublic: process.env.MAKE_PUBLIC === "true", @@ -88,7 +89,7 @@ export const config = { ), customFilterPrompt: process.env.CUSTOM_FILTER_PROMPT || null, placeholderImagePath: process.env.PLACEHOLDER_IMAGE_PATH || null, - backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3, + // backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3, }; export type Config = typeof config; diff --git a/storage-resize-images/functions/src/index.ts b/storage-resize-images/functions/src/index.ts index 9a5cd8f86..5cb7034f5 100644 --- a/storage-resize-images/functions/src/index.ts +++ b/storage-resize-images/functions/src/index.ts @@ -156,80 +156,80 @@ export const generateResizedImage = functions.storage /** * */ -export const backfillResizedImages = functions.tasks - .taskQueue() - .onDispatch(async (data) => { - const runtime = getExtensions().runtime(); - if (!config.doBackfill) { - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Existing images were not resized because 'Backfill existing images' was configured to false." + - " If you want to resize existing images, reconfigure this instance." - ); - return; - } - if (data?.nextPageQuery == undefined) { - logs.startBackfill(); - } - - const bucket = admin.storage().bucket(process.env.IMG_BUCKET); - const query = data.nextPageQuery || { - autoPaginate: false, - maxResults: config.backfillBatchSize, - }; - const [files, nextPageQuery] = await bucket.getFiles(query); - const filesToResize = files.filter((f: File) => { - logs.continueBackfill(f.metadata.name); - return shouldResize(convertToObjectMetadata(f.metadata)); - }); - - const filePromises = filesToResize.map((f) => { - return generateResizedImageHandler( - convertToObjectMetadata(f.metadata), - /*verbose=*/ false - ); - }); - const results = await Promise.allSettled(filePromises); - - const pageErrorsCount = results.filter( - (r) => r.status === "rejected" - ).length; - const pageSuccessCount = results.filter( - (r) => r.status === "fulfilled" - ).length; - const oldErrorsCount = Number(data.errorsCount) || 0; - const oldSuccessCount = Number(data.successCount) || 0; - const errorsCount = pageErrorsCount + oldErrorsCount; - const successCount = pageSuccessCount + oldSuccessCount; - - if (nextPageQuery) { - const queue = getFunctions().taskQueue( - `locations/${config.location}/functions/backfillResizedImages`, - process.env.EXT_INSTANCE_ID - ); - await queue.enqueue({ - nextPageQuery, - errorsCount, - successCount, - }); - } else { - logs.backfillComplete(successCount, errorsCount); - if (errorsCount == 0) { - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - `Successfully resized ${successCount} images.` - ); - } else if (errorsCount > 0 && successCount > 0) { - await runtime.setProcessingState( - "PROCESSING_WARNING", - `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` - ); - } - if (errorsCount > 0 && successCount == 0) { - await runtime.setProcessingState( - "PROCESSING_FAILED", - `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` - ); - } - } - }); +// export const backfillResizedImages = functions.tasks +// .taskQueue() +// .onDispatch(async (data) => { +// const runtime = getExtensions().runtime(); +// if (!config.doBackfill) { +// await runtime.setProcessingState( +// "PROCESSING_COMPLETE", +// "Existing images were not resized because 'Backfill existing images' was configured to false." + +// " If you want to resize existing images, reconfigure this instance." +// ); +// return; +// } +// if (data?.nextPageQuery == undefined) { +// logs.startBackfill(); +// } + +// const bucket = admin.storage().bucket(process.env.IMG_BUCKET); +// const query = data.nextPageQuery || { +// autoPaginate: false, +// maxResults: config.backfillBatchSize, +// }; +// const [files, nextPageQuery] = await bucket.getFiles(query); +// const filesToResize = files.filter((f: File) => { +// logs.continueBackfill(f.metadata.name); +// return shouldResize(convertToObjectMetadata(f.metadata)); +// }); + +// const filePromises = filesToResize.map((f) => { +// return generateResizedImageHandler( +// convertToObjectMetadata(f.metadata), +// /*verbose=*/ false +// ); +// }); +// const results = await Promise.allSettled(filePromises); + +// const pageErrorsCount = results.filter( +// (r) => r.status === "rejected" +// ).length; +// const pageSuccessCount = results.filter( +// (r) => r.status === "fulfilled" +// ).length; +// const oldErrorsCount = Number(data.errorsCount) || 0; +// const oldSuccessCount = Number(data.successCount) || 0; +// const errorsCount = pageErrorsCount + oldErrorsCount; +// const successCount = pageSuccessCount + oldSuccessCount; + +// if (nextPageQuery) { +// const queue = getFunctions().taskQueue( +// `locations/${config.location}/functions/backfillResizedImages`, +// process.env.EXT_INSTANCE_ID +// ); +// await queue.enqueue({ +// nextPageQuery, +// errorsCount, +// successCount, +// }); +// } else { +// logs.backfillComplete(successCount, errorsCount); +// if (errorsCount == 0) { +// await runtime.setProcessingState( +// "PROCESSING_COMPLETE", +// `Successfully resized ${successCount} images.` +// ); +// } else if (errorsCount > 0 && successCount > 0) { +// await runtime.setProcessingState( +// "PROCESSING_WARNING", +// `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` +// ); +// } +// if (errorsCount > 0 && successCount == 0) { +// await runtime.setProcessingState( +// "PROCESSING_FAILED", +// `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` +// ); +// } +// } +// });