Release 20/03/25

.github/workflows/npm_publish_bq_scripts.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
       - name: NPM install
         run: npm install
       - name: Publish BigQuery Schema Views

.github/workflows/readmes-updated.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 22
           cache: "npm"
           cache-dependency-path: "**/functions/package-lock.json"
 

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: ["18"]
+        node: ["20"]
     name: node.js_${{ matrix.node }}_test
     steps:
       - uses: actions/checkout@v3

.github/workflows/validate.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
       - name: NPM install
         run: SKIP_POSTINSTALL=yes npm i
       - name: Prettier Lint Check

firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md

@@ -140,6 +140,22 @@ To retry the failed imports, you can use the output file to manually inspect or
 
 > **Note:** If the specified file already exists, it will be **cleared** before writing new failed batch paths.
 
+### Using the "Generate Schema Views" After Import
+
+After using fs-bq-import-collection to import your Firestore data to BigQuery, your data will be available in two forms: a 'raw changelog' table that streams all Firestore events chronologically, and a 'raw latest' view showing the current state of each document. However, the raw data doesn't have proper typing; all fields are stored as strings inside a JSON structure. To make this data more useful for querying, you should generate schema views.
+
+#### Why Use Schema Views
+
+**Proper Data Types**: Convert string-based JSON to properly typed BigQuery columns.
+**Easier Querying**: Query your data using column names rather than JSON functions.
+**Preserve Complex Types**: Handle Firestore-specific types like arrays, maps, and geopoints.
+
+#### Guide For Generate Schema Views
+
+To generate a schema view, you may use the official fs-bq-schema-views CLI tool. You can find a guide for using this tool [here](./GENERATE_SCHEMA_VIEWS.md).
+
+This Generate Schema Views tool has an optional AI schema generation tool, powered by Gemini, where it can sample from your original Cloud Firestore collection and generate an appropriate schema for your BigQuery Views as a first step. You can review and customize this schema before applying it to BigQuery.
+
 ### Using a Transform Function
 
 You can optionally provide a transform function URL (`--transform-function-url` or `-f`) that will transform document data before it's written to BigQuery. The transform function should should recieve document data and return transformed data. The payload will contain the following:

firestore-shorten-urls-bitly/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Version 0.2.0
+
+feat: use v2 firestore trigger
+
+feat: allow non-(default) firestore instances
+
 ## Version 0.1.18
 
 feat - move to Node.js 20 runtimes

firestore-shorten-urls-bitly/README.md

@@ -49,11 +49,14 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan
 * Short URL field name: What is the name of the field where you want to store your shortened URLs?
 
 
+* Firestore Database: The Firestore database to use. Use "(default)" for the default database.
 
 
-**Cloud Functions:**
 
-* **fsurlshortener:** Listens for writes of new URLs to your specified Cloud Firestore collection, shortens the URLs, then writes the shortened form back to the same document.
+
+**Other Resources**:
+
+* fsurlshortener (firebaseextensions.v1beta.v2function)
 
 
 

firestore-shorten-urls-bitly/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-shorten-urls-bitly
-version: 0.1.18
+version: 0.2.0
 specVersion: v1beta
 
 displayName: Shorten URLs in Firestore
@@ -48,16 +48,26 @@ roles:
 
 resources:
   - name: fsurlshortener
-    type: firebaseextensions.v1beta.function
+    type: firebaseextensions.v1beta.v2function
     description:
       Listens for writes of new URLs to your specified Cloud Firestore
       collection, shortens the URLs, then writes the shortened form back to the
       same document.
     properties:
-      runtime: nodejs20
+      sourceDirectory: functions
+      buildConfig:
+        runtime: nodejs22
+      serviceConfig:
+        timeoutSeconds: 120
       eventTrigger:
-        eventType: providers/cloud.firestore/eventTypes/document.write
-        resource: projects/${param:PROJECT_ID}/databases/(default)/documents/${param:COLLECTION_PATH}/{documentId}
+        eventType: google.cloud.firestore.document.v1.written
+        triggerRegion: ${LOCATION}
+        eventFilters:
+          - attribute: database
+            value: ${DATABASE}
+          - attribute: document
+            value: ${COLLECTION_PATH}/{documentId}
+            operator: match-path-pattern
 
 externalServices:
   - name: Bitly
@@ -101,16 +111,22 @@ params:
     default: shortUrl
     required: true
 
+  - param: DATABASE
+    label: Firestore Database
+    description: >
+      The Firestore database to use. Use "(default)" for the default database.
+    example: (default)
+    default: (default)
+    required: true
+
 events:
   - type: firebase.extensions.firestore-shorten-urls-bitly.v1.onStart
     description:
       Occurs when a trigger has been called within the Extension, and will
       include data such as the context of the trigger request.
 
   - type: firebase.extensions.firestore-shorten-urls-bitly.v1.onSuccess
-    description:
-      Occurs when image resizing completes successfully. The event will contain
-      further details about specific formats and sizes.
+    description: Occurs when URL shortening completes successfully.
 
   - type: firebase.extensions.firestore-shorten-urls-bitly.v1.onError
     description:

firestore-shorten-urls-bitly/functions/src/abstract-shortener.ts

@@ -14,11 +14,18 @@
  * limitations under the License.
  */
 
-import * as admin from "firebase-admin";
-import * as functions from "firebase-functions";
+import { initializeApp } from "firebase-admin/app";
+import {
+  getFirestore,
+  DocumentSnapshot,
+  FieldValue,
+} from "firebase-admin/firestore";
+import { Change } from "firebase-functions/v2/firestore";
+import { logger } from "firebase-functions/v2";
 
 import * as logs from "./logs";
 import * as events from "./events";
+import config from "./config";
 
 enum ChangeType {
   CREATE,
@@ -35,14 +42,10 @@ export abstract class FirestoreUrlShortener {
   ) {
     this.urlFieldName = urlFieldName;
     this.shortUrlFieldName = shortUrlFieldName;
-
-    // Initialize the Firebase Admin SDK
-    admin.initializeApp();
+    initializeApp();
   }
 
-  public async onDocumentWrite(
-    change: functions.Change<admin.firestore.DocumentSnapshot>
-  ) {
+  public async onDocumentWrite(change: Change<DocumentSnapshot>) {
     this.logs.start();
 
     if (this.urlFieldName === this.shortUrlFieldName) {
@@ -52,33 +55,40 @@ export abstract class FirestoreUrlShortener {
 
     const changeType = this.getChangeType(change);
 
-    switch (changeType) {
-      case ChangeType.CREATE:
-        await this.handleCreateDocument(change.after);
-        break;
-      case ChangeType.DELETE:
-        this.handleDeleteDocument();
-        break;
-      case ChangeType.UPDATE:
-        await this.handleUpdateDocument(change.before, change.after);
-        break;
-      default: {
-        const err = new Error(`Invalid change type: ${changeType}`);
-        await events.recordErrorEvent(err);
-        throw err;
+    try {
+      switch (changeType) {
+        case ChangeType.CREATE:
+          await this.handleCreateDocument(change.after);
+          break;
+        case ChangeType.DELETE:
+          this.handleDeleteDocument();
+          break;
+        case ChangeType.UPDATE:
+          await this.handleUpdateDocument(change.before, change.after);
+          break;
+        default: {
+          const err = new Error(`Invalid change type: ${changeType}`);
+          await events.recordErrorEvent(err);
+          throw err;
+        }
       }
-    }
 
-    this.logs.complete();
+      this.logs.complete();
+    } catch (err) {
+      logger.error("Error in extension execution", err);
+      await events.recordErrorEvent(
+        err instanceof Error ? err : new Error(String(err))
+      );
+      throw err;
+    }
   }
 
-  protected extractUrl(snapshot: admin.firestore.DocumentSnapshot) {
-    return snapshot.get(this.urlFieldName);
+  protected extractUrl(snapshot: DocumentSnapshot) {
+    const data = snapshot.data();
+    return data ? data[this.urlFieldName] : undefined;
   }
 
-  private getChangeType(
-    change: functions.Change<admin.firestore.DocumentSnapshot>
-  ) {
+  private getChangeType(change: Change<DocumentSnapshot>) {
     if (!change.after.exists) {
       return ChangeType.DELETE;
     }
@@ -88,9 +98,7 @@ export abstract class FirestoreUrlShortener {
     return ChangeType.UPDATE;
   }
 
-  private async handleCreateDocument(
-    snapshot: admin.firestore.DocumentSnapshot
-  ) {
+  private async handleCreateDocument(snapshot: DocumentSnapshot) {
     const url = this.extractUrl(snapshot);
     if (url) {
       this.logs.documentCreatedWithUrl();
@@ -105,8 +113,8 @@ export abstract class FirestoreUrlShortener {
   }
 
   private async handleUpdateDocument(
-    before: admin.firestore.DocumentSnapshot,
-    after: admin.firestore.DocumentSnapshot
+    before: DocumentSnapshot,
+    after: DocumentSnapshot
   ) {
     const urlAfter = this.extractUrl(after);
     const urlBefore = this.extractUrl(before);
@@ -118,27 +126,27 @@ export abstract class FirestoreUrlShortener {
       await this.shortenUrl(after);
     } else if (urlBefore) {
       this.logs.documentUpdatedDeletedUrl();
-      await this.updateShortUrl(after, admin.firestore.FieldValue.delete());
+      await this.updateShortUrl(after, FieldValue.delete());
     } else {
       this.logs.documentUpdatedNoUrl();
     }
   }
 
-  protected abstract shortenUrl(
-    snapshot: admin.firestore.DocumentSnapshot
-  ): Promise<void>;
+  protected abstract shortenUrl(snapshot: DocumentSnapshot): Promise<void>;
 
   protected async updateShortUrl(
-    snapshot: admin.firestore.DocumentSnapshot,
+    snapshot: DocumentSnapshot,
     url: any
   ): Promise<void> {
     this.logs.updateDocument(snapshot.ref.path);
 
-    // Wrapping in transaction to allow for automatic retries (#48)
-    await admin.firestore().runTransaction((transaction) => {
+    // Wrapping in transaction to allow for automatic retries
+    const firestore = getFirestore(config.database);
+    await firestore.runTransaction((transaction) => {
       transaction.update(snapshot.ref, this.shortUrlFieldName, url);
       return Promise.resolve();
     });
+
     this.logs.updateDocumentComplete(snapshot.ref.path);
     await events.recordSuccessEvent({
       subject: snapshot.ref.path,

firestore-shorten-urls-bitly/functions/src/config.ts

@@ -20,4 +20,5 @@ export default {
   location: process.env.LOCATION,
   shortUrlFieldName: process.env.SHORT_URL_FIELD_NAME,
   urlFieldName: process.env.URL_FIELD_NAME,
+  database: process.env.DATABASE,
 };

firestore-shorten-urls-bitly/functions/src/index.ts

@@ -13,9 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-import * as admin from "firebase-admin";
-import * as functions from "firebase-functions";
+import { DocumentSnapshot } from "firebase-admin/firestore";
+import { onDocumentWritten } from "firebase-functions/v2/firestore";
 
 import { FirestoreUrlShortener } from "./abstract-shortener";
 import config from "./config";
@@ -39,9 +38,7 @@ class FirestoreBitlyUrlShortener extends FirestoreUrlShortener {
     logs.init();
   }
 
-  protected async shortenUrl(
-    snapshot: admin.firestore.DocumentSnapshot
-  ): Promise<void> {
+  protected async shortenUrl(snapshot: DocumentSnapshot): Promise<void> {
     const url = this.extractUrl(snapshot);
     logs.shortenUrl(url);
 
@@ -81,10 +78,13 @@ const urlShortener = new FirestoreBitlyUrlShortener(
 
 events.setupEventChannel();
 
-export const fsurlshortener = functions.firestore
-  .document(config.collectionPath)
-  .onWrite(async (change, context) => {
-    await events.recordStartEvent({ context, change });
-    await urlShortener.onDocumentWrite(change);
+export const fsurlshortener = onDocumentWritten(
+  `${config.collectionPath}/{documentId}`,
+  async (event) => {
+    const { data, ...context } = event;
+
+    await events.recordStartEvent({ context, change: data });
+    await urlShortener.onDocumentWrite(data);
     await events.recordCompletionEvent({ context });
-  });
+  }
+);

storage-resize-images/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Version 0.2.9
+
+fixed - run an npm audit and bump vulnerable dependencies
+
+fixed - use Promise.allSettled to await various resizes
+
 ## Version 0.2.8
 
 fixed - support '+' character in paths

storage-resize-images/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: storage-resize-images
-version: 0.2.8
+version: 0.2.9
 specVersion: v1beta
 
 displayName: Resize Images

storage-resize-images/functions/__tests__/convert-image.test.ts

@@ -1,5 +1,5 @@
 import mockedEnv from "mocked-env";
-import imageType from "image-type";
+const imageType = require("image-type");
 import * as util from "util";
 import * as fs from "fs";
 import * as path from "path";