From a3049a1a5bfb95e712d9c2dadc9c1fd1b30e1e1c Mon Sep 17 00:00:00 2001 From: Corie Watson Date: Fri, 14 Mar 2025 16:06:23 +0000 Subject: [PATCH] feat(fs-bq-change-tracker): add logger with log levels (#2331) * feat(fs-bq-change-tracker): add logger with diff levels * chore(fs-bq-change-tracker): address feedback --- .../package.json | 2 +- .../src/__tests__/fixtures/changeTracker.ts | 3 + .../src/__tests__/logger.test.ts | 66 ++++++++++++++++ .../src/bigquery/index.ts | 4 + .../src/index.ts | 1 + .../src/logger.ts | 76 +++++++++++++++++++ .../src/logs.ts | 60 +++++++-------- 7 files changed, 181 insertions(+), 31 deletions(-) create mode 100644 firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts create mode 100644 firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json index b91228c69..cc15e39ea 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json @@ -5,7 +5,7 @@ "url": "github.com/firebase/extensions.git", "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" }, - "version": "1.1.39", + "version": "1.1.40", "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", "main": "./lib/index.js", "scripts": { diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts index 3bcc9b060..206153429 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts @@ -4,6 +4,7 @@ import { FirestoreBigQueryEventHistoryTracker, FirestoreDocumentChangeEvent, } from "../.."; +import { LogLevel } from "../../logger"; export const changeTracker = ({ datasetId = "", @@ -22,6 +23,7 @@ export const changeTracker = ({ useIncrementalMaterializedView = false, maxStaleness = undefined, refreshIntervalMinutes = undefined, + logLevel = LogLevel.INFO, }): FirestoreBigQueryEventHistoryTracker => { return new FirestoreBigQueryEventHistoryTracker({ datasetId, @@ -40,6 +42,7 @@ export const changeTracker = ({ useIncrementalMaterializedView, maxStaleness, refreshIntervalMinutes, + logLevel, }); }; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts new file mode 100644 index 000000000..8a056d84d --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts @@ -0,0 +1,66 @@ +import { Logger, LogLevel, logger as defaultLogger } from "../logger"; +import { logger as funcsLogger } from "firebase-functions"; + +// Mock firebase-functions logger functions +jest.mock("firebase-functions", () => ({ + logger: { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +})); + +describe("Logger", () => { + let log: Logger; + + beforeEach(() => { + // Create a new instance before each test + log = new Logger(LogLevel.DEBUG); + jest.clearAllMocks(); + }); + + describe("log level methods", () => { + test("debug should call funcsLogger.debug when level is DEBUG", () => { + log.setLogLevel(LogLevel.DEBUG); + log.debug("debug message"); + expect(funcsLogger.debug).toHaveBeenCalledWith("debug message"); + }); + + test("debug should not call funcsLogger.debug when level is INFO", () => { + log.setLogLevel(LogLevel.INFO); + log.debug("debug message"); + expect(funcsLogger.debug).not.toHaveBeenCalled(); + }); + + test("info should call funcsLogger.info when level is INFO or lower", () => { + log.setLogLevel(LogLevel.INFO); + log.info("info message"); + expect(funcsLogger.info).toHaveBeenCalledWith("info message"); + }); + + test("warn should call funcsLogger.warn when level is WARN or lower", () => { + log.setLogLevel(LogLevel.WARN); + log.warn("warn message"); + expect(funcsLogger.warn).toHaveBeenCalledWith("warn message"); + }); + + test("error should call funcsLogger.error when level is ERROR or lower", () => { + log.setLogLevel(LogLevel.ERROR); + log.error("error message"); + expect(funcsLogger.error).toHaveBeenCalledWith("error message"); + }); + + test("no logging should occur when log level is SILENT", () => { + log.setLogLevel(LogLevel.SILENT); + log.debug("debug message"); + log.info("info message"); + log.warn("warn message"); + log.error("error message"); + expect(funcsLogger.debug).not.toHaveBeenCalled(); + expect(funcsLogger.info).not.toHaveBeenCalled(); + expect(funcsLogger.warn).not.toHaveBeenCalled(); + expect(funcsLogger.error).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index 5b84be835..af6423ea5 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -44,6 +44,7 @@ import { Clustering } from "./clustering"; import { tableRequiresUpdate } from "./checkUpdates"; import { parseErrorMessage, waitForInitialization } from "./utils"; import { initializeLatestView } from "./initializeLatestView"; +import { logger, LogLevel } from "../logger"; export { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; @@ -68,6 +69,7 @@ export interface FirestoreBigQueryEventHistoryTrackerConfig { useIncrementalMaterializedView?: boolean; maxStaleness?: string; refreshIntervalMinutes?: number; + logLevel?: LogLevel | string; } /** @@ -94,6 +96,8 @@ export class FirestoreBigQueryEventHistoryTracker if (!this.config.datasetLocation) { this.config.datasetLocation = "us"; } + + logger.setLogLevel(this.config.logLevel || LogLevel.INFO); } async record(events: FirestoreDocumentChangeEvent[]) { diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts index d9cc7ffa1..858c256ab 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts @@ -24,3 +24,4 @@ export { FirestoreDocumentChangeEvent, FirestoreEventHistoryTracker, } from "./tracker"; +export { LogLevel, Logger } from "./logger"; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts new file mode 100644 index 000000000..76456ffc9 --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { logger as funcsLogger } from "firebase-functions"; + +export enum LogLevel { + DEBUG = "debug", // Will log everything + INFO = "info", // Will log info, warnings, and errors + WARN = "warn", // Will log warnings and errors + ERROR = "error", // Will log errors only + SILENT = "silent", // Won't log anything +} + +const levels = { + debug: 0, + info: 1, + warn: 2, + error: 3, + silent: 4, +}; + +export class Logger { + private logLevel: number; + + constructor(logLevel: LogLevel | string = LogLevel.INFO) { + this.setLogLevel(logLevel); + } + + setLogLevel(logLevel: LogLevel | string): void { + if (typeof logLevel === "string") { + this.logLevel = levels[logLevel as keyof typeof levels] ?? levels.info; + } else { + this.logLevel = levels[logLevel]; + } + } + + debug(...args: any[]): void { + this.runIfLogLevel(levels.debug, funcsLogger.debug, ...args); + } + + info(...args: any[]): void { + this.runIfLogLevel(levels.info, funcsLogger.info, ...args); + } + + warn(...args: any[]): void { + this.runIfLogLevel(levels.warn, funcsLogger.warn, ...args); + } + + error(...args: any[]): void { + this.runIfLogLevel(levels.error, funcsLogger.error, ...args); + } + + log(...args: any[]): void { + this.info(...args); + } + + private runIfLogLevel(level: number, func: Function, ...args: any[]): void { + if (this.logLevel <= level) { + func(...args); + } + } +} + +export const logger = new Logger(); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts index aed6b1562..9b1b2f658 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts @@ -16,22 +16,22 @@ import { Table } from "@google-cloud/bigquery"; import { firestore } from "firebase-admin"; -import { logger } from "firebase-functions"; +import { logger } from "./logger"; export const arrayFieldInvalid = (fieldName: string) => { logger.warn(`Array field '${fieldName}' does not contain an array, skipping`); }; export const bigQueryDatasetCreated = (datasetId: string) => { - logger.log(`Created BigQuery dataset: ${datasetId}`); + logger.info(`Created BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetCreating = (datasetId: string) => { - logger.log(`Creating BigQuery dataset: ${datasetId}`); + logger.debug(`Creating BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetExists = (datasetId: string) => { - logger.log(`BigQuery dataset already exists: ${datasetId}`); + logger.info(`BigQuery dataset already exists: ${datasetId}`); }; export const bigQueryErrorRecordingDocumentChange = (e: Error) => { @@ -39,112 +39,112 @@ export const bigQueryErrorRecordingDocumentChange = (e: Error) => { }; export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => { - logger.log(`BigQuery latest snapshot view query:\n${query}`); + logger.debug(`BigQuery latest snapshot view query:\n${query}`); }; export const bigQuerySchemaViewCreated = (name: string) => { - logger.log(`BigQuery created schema view ${name}\n`); + logger.debug(`BigQuery created schema view ${name}\n`); }; export const bigQueryTableAlreadyExists = ( tableName: string, datasetName: string ) => { - logger.log( + logger.debug( `BigQuery table with name ${tableName} already ` + `exists in dataset ${datasetName}!` ); }; export const bigQueryTableCreated = (tableName: string) => { - logger.log(`Created BigQuery table: ${tableName}`); + logger.info(`Created BigQuery table: ${tableName}`); }; export const bigQueryTableCreating = (tableName: string) => { - logger.log(`Creating BigQuery table: ${tableName}`); + logger.debug(`Creating BigQuery table: ${tableName}`); }; export const bigQueryTableUpdated = (tableName: string) => { - logger.log(`Updated existing BigQuery table: ${tableName}`); + logger.info(`Updated existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpdating = (tableName: string) => { - logger.log(`Updating existing BigQuery table: ${tableName}`); + logger.debug(`Updating existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpToDate = (tableName: string) => { - logger.log(`BigQuery table: ${tableName} is up to date`); + logger.info(`BigQuery table: ${tableName} is up to date`); }; export const bigQueryTableValidated = (tableName: string) => { - logger.log(`Validated existing BigQuery table: ${tableName}`); + logger.info(`Validated existing BigQuery table: ${tableName}`); }; export const bigQueryTableValidating = (tableName: string) => { - logger.log(`Validating existing BigQuery table: ${tableName}`); + logger.debug(`Validating existing BigQuery table: ${tableName}`); }; export const bigQueryUserDefinedFunctionCreating = (functionName: string) => { - logger.log(`Creating BigQuery user-defined function ${functionName}`); + logger.debug(`Creating BigQuery user-defined function ${functionName}`); }; export const bigQueryUserDefinedFunctionCreated = (functionName: string) => { - logger.log(`Created BigQuery user-defined function ${functionName}`); + logger.info(`Created BigQuery user-defined function ${functionName}`); }; export const bigQueryViewCreated = (viewName: string) => { - logger.log(`Created BigQuery view: ${viewName}`); + logger.info(`Created BigQuery view: ${viewName}`); }; export const bigQueryViewCreating = (viewName: string, query: string) => { - logger.log(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`); + logger.debug(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`); }; export const bigQueryViewAlreadyExists = ( viewName: string, datasetName: string ) => { - logger.log( + logger.info( `View with id ${viewName} already exists in dataset ${datasetName}.` ); }; export const bigQueryViewUpdated = (viewName: string) => { - logger.log(`Updated existing BigQuery view: ${viewName}`); + logger.info(`Updated existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpdating = (viewName: string) => { - logger.log(`Updating existing BigQuery view: ${viewName}`); + logger.debug(`Updating existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpToDate = (viewName: string) => { - logger.log(`BigQuery view: ${viewName} is up to date`); + logger.info(`BigQuery view: ${viewName} is up to date`); }; export const bigQueryViewValidated = (viewName: string) => { - logger.log(`Validated existing BigQuery view: ${viewName}`); + logger.info(`Validated existing BigQuery view: ${viewName}`); }; export const bigQueryViewValidating = (viewName: string) => { - logger.log(`Validating existing BigQuery view: ${viewName}`); + logger.debug(`Validating existing BigQuery view: ${viewName}`); }; export const complete = () => { - logger.log("Completed mod execution"); + logger.info("Completed mod execution"); }; export const dataInserted = (rowCount: number) => { - logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`); + logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`); }; export const dataInsertRetried = (rowCount: number) => { - logger.log( + logger.debug( `Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring unknown columns)` ); }; export const dataInserting = (rowCount: number) => { - logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`); + logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`); }; export const dataTypeInvalid = ( @@ -168,11 +168,11 @@ export const timestampMissingValue = (fieldName: string) => { }; export const addNewColumn = (table: string, field: string) => { - logger.log(`Updated '${table}' table with a '${field}' column`); + logger.info(`Updated '${table}' table with a '${field}' column`); }; export const addPartitionFieldColumn = (table, field) => { - logger.log( + logger.info( `Updated '${table}' table with a partition field '${field}' column` ); };