diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 23c0f4f8d5..1032320d6c 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91739,27 +91739,28 @@ var fs13 = __toESM(require("fs")); async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetails, features, logger) { if (getRequiredInput("upload-database") !== "true") { logger.debug("Database upload disabled in workflow. Skipping upload."); - return; + return []; } if (!config.analysisKinds.includes("code-scanning" /* CodeScanning */)) { logger.debug( `Not uploading database because 'analysis-kinds: ${"code-scanning" /* CodeScanning */}' is not enabled.` ); - return; + return []; } if (isInTestMode()) { logger.debug("In test mode. Skipping database upload."); - return; + return []; } if (config.gitHubVersion.type !== "GitHub.com" /* DOTCOM */ && config.gitHubVersion.type !== "GitHub Enterprise Cloud with data residency" /* GHEC_DR */) { logger.debug("Not running against github.com or GHEC-DR. Skipping upload."); - return; + return []; } if (!await isAnalyzingDefaultBranch()) { logger.debug("Not analyzing default branch. Skipping upload."); - return; + return []; } - const cleanupLevel = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */) ? "overlay" /* Overlay */ : "clear" /* Clear */; + const shouldUploadOverlayBase = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */); + const cleanupLevel = shouldUploadOverlayBase ? "overlay" /* Overlay */ : "clear" /* Clear */; await withGroupAsync("Cleaning up databases", async () => { await codeql.databaseCleanupCluster(config, cleanupLevel); }); @@ -91770,6 +91771,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai if (uploadsBaseUrl.endsWith("/")) { uploadsBaseUrl = uploadsBaseUrl.slice(0, -1); } + const reports = []; for (const language of config.languages) { try { const bundledDb = await bundleDb(config, language, codeql, language); @@ -91779,6 +91781,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai getRequiredInput("checkout_path") ); try { + const startTime = performance.now(); await client.request( `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, { @@ -91796,14 +91799,28 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai } } ); + const endTime = performance.now(); + reports.push({ + language, + zipped_upload_size_bytes: bundledDbSize, + is_overlay_base: shouldUploadOverlayBase, + upload_duration_ms: endTime - startTime + }); logger.debug(`Successfully uploaded database for ${language}`); } finally { bundledDbReadStream.close(); } } catch (e) { - logger.warning(`Failed to upload database for ${language}: ${e}`); + logger.warning( + `Failed to upload database for ${language}: ${getErrorMessage(e)}` + ); + reports.push({ + language, + error: getErrorMessage(e) + }); } } + return reports; } // src/status-report.ts @@ -93817,7 +93834,7 @@ async function postProcessAndUploadSarif(logger, features, uploadKind, checkoutP } // src/analyze-action.ts -async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, dependencyCacheResults, logger) { +async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, dependencyCacheResults, databaseUploadResults, logger) { const status = getActionsStatus(error3, stats?.analyze_failure_language); const statusReportBase = await createStatusReportBase( "finish" /* Analyze */, @@ -93835,7 +93852,8 @@ async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUplo ...stats || {}, ...dbCreationTimings || {}, ...trapCacheCleanup || {}, - dependency_caching_upload_results: dependencyCacheResults + dependency_caching_upload_results: dependencyCacheResults, + database_upload_results: databaseUploadResults }; if (config && didUploadTrapCaches) { const trapCacheUploadStatusReport = { @@ -93917,6 +93935,7 @@ async function run() { let dbCreationTimings = void 0; let didUploadTrapCaches = false; let dependencyCacheResults; + let databaseUploadResults = []; initializeEnvironment(getActionVersion()); persistInputs(); const logger = getActionsLogger(); @@ -94046,7 +94065,7 @@ async function run() { logger.info("Not uploading results"); } await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger); - await cleanupAndUploadDatabases( + databaseUploadResults = await cleanupAndUploadDatabases( repositoryNwo, codeql, config, @@ -94100,6 +94119,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger ); return; @@ -94118,6 +94138,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger ); } else if (runStats !== void 0) { @@ -94131,6 +94152,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger ); } else { @@ -94144,6 +94166,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger ); } diff --git a/src/analyze-action.ts b/src/analyze-action.ts index c6a3d7dc26..6f45ee5f17 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -20,7 +20,10 @@ import { runAutobuild } from "./autobuild"; import { getTotalCacheSize, shouldStoreCache } from "./caching-utils"; import { getCodeQL } from "./codeql"; import { Config, getConfig } from "./config-utils"; -import { cleanupAndUploadDatabases } from "./database-upload"; +import { + cleanupAndUploadDatabases, + DatabaseUploadResult, +} from "./database-upload"; import { DependencyCacheUploadStatusReport, uploadDependencyCaches, @@ -54,15 +57,13 @@ interface AnalysisStatusReport extends uploadLib.UploadStatusReport, QueriesStatusReport {} -interface DependencyCachingUploadStatusReport { - dependency_caching_upload_results?: DependencyCacheUploadStatusReport; -} - interface FinishStatusReport extends StatusReportBase, DatabaseCreationTimings, - AnalysisStatusReport, - DependencyCachingUploadStatusReport {} + AnalysisStatusReport { + dependency_caching_upload_results?: DependencyCacheUploadStatusReport; + database_upload_results: DatabaseUploadResult[]; +} interface FinishWithTrapUploadStatusReport extends FinishStatusReport { /** Size of TRAP caches that we uploaded, in bytes. */ @@ -81,6 +82,7 @@ async function sendStatusReport( didUploadTrapCaches: boolean, trapCacheCleanup: TrapCacheCleanupStatusReport | undefined, dependencyCacheResults: DependencyCacheUploadStatusReport | undefined, + databaseUploadResults: DatabaseUploadResult[], logger: Logger, ) { const status = getActionsStatus(error, stats?.analyze_failure_language); @@ -101,6 +103,7 @@ async function sendStatusReport( ...(dbCreationTimings || {}), ...(trapCacheCleanup || {}), dependency_caching_upload_results: dependencyCacheResults, + database_upload_results: databaseUploadResults, }; if (config && didUploadTrapCaches) { const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = { @@ -218,6 +221,7 @@ async function run() { let dbCreationTimings: DatabaseCreationTimings | undefined = undefined; let didUploadTrapCaches = false; let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined; + let databaseUploadResults: DatabaseUploadResult[] = []; util.initializeEnvironment(actionsUtil.getActionVersion()); // Make inputs accessible in the `post` step, details at @@ -389,7 +393,7 @@ async function run() { // Possibly upload the database bundles for remote queries. // Note: Take care with the ordering of this call since databases may be cleaned up // at the `overlay` or `clear` level. - await cleanupAndUploadDatabases( + databaseUploadResults = await cleanupAndUploadDatabases( repositoryNwo, codeql, config, @@ -461,6 +465,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger, ); return; @@ -483,6 +488,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger, ); } else if (runStats !== undefined) { @@ -496,6 +502,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger, ); } else { @@ -509,6 +516,7 @@ async function run() { didUploadTrapCaches, trapCacheCleanupTelemetry, dependencyCacheResults, + databaseUploadResults, logger, ); } diff --git a/src/database-upload.test.ts b/src/database-upload.test.ts index e07ff1da2e..57d07adb28 100644 --- a/src/database-upload.test.ts +++ b/src/database-upload.test.ts @@ -231,7 +231,7 @@ test("Don't crash if uploading a database fails", async (t) => { (v) => v.type === "warning" && v.message === - "Failed to upload database for javascript: Error: some error message", + "Failed to upload database for javascript: some error message", ) !== undefined, ); }); diff --git a/src/database-upload.ts b/src/database-upload.ts index ab0e2636a2..c3e603285f 100644 --- a/src/database-upload.ts +++ b/src/database-upload.ts @@ -13,6 +13,20 @@ import { RepositoryNwo } from "./repository"; import * as util from "./util"; import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util"; +/** Information about a database upload. */ +export interface DatabaseUploadResult { + /** Language of the database. */ + language: string; + /** Size of the zipped database in bytes. */ + zipped_upload_size_bytes?: number; + /** Whether the uploaded database is an overlay base. */ + is_overlay_base?: boolean; + /** Time taken to upload database in milliseconds. */ + upload_duration_ms?: number; + /** If there was an error during database upload, this is its message. */ + error?: string; +} + export async function cleanupAndUploadDatabases( repositoryNwo: RepositoryNwo, codeql: CodeQL, @@ -20,22 +34,22 @@ export async function cleanupAndUploadDatabases( apiDetails: GitHubApiDetails, features: FeatureEnablement, logger: Logger, -): Promise { +): Promise { if (actionsUtil.getRequiredInput("upload-database") !== "true") { logger.debug("Database upload disabled in workflow. Skipping upload."); - return; + return []; } if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) { logger.debug( `Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`, ); - return; + return []; } if (util.isInTestMode()) { logger.debug("In test mode. Skipping database upload."); - return; + return []; } // Do nothing when not running against github.com @@ -44,20 +58,22 @@ export async function cleanupAndUploadDatabases( config.gitHubVersion.type !== util.GitHubVariant.GHEC_DR ) { logger.debug("Not running against github.com or GHEC-DR. Skipping upload."); - return; + return []; } if (!(await gitUtils.isAnalyzingDefaultBranch())) { // We only want to upload a database if we are analyzing the default branch. logger.debug("Not analyzing default branch. Skipping upload."); - return; + return []; } - const cleanupLevel = + // If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages. + const shouldUploadOverlayBase = config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase && - (await features.getValue(Feature.UploadOverlayDbToApi)) - ? CleanupLevel.Overlay - : CleanupLevel.Clear; + (await features.getValue(Feature.UploadOverlayDbToApi)); + const cleanupLevel = shouldUploadOverlayBase + ? CleanupLevel.Overlay + : CleanupLevel.Clear; // Clean up the database, since intermediate results may still be written to the // database if there is high RAM pressure. @@ -77,6 +93,7 @@ export async function cleanupAndUploadDatabases( uploadsBaseUrl = uploadsBaseUrl.slice(0, -1); } + const reports: DatabaseUploadResult[] = []; for (const language of config.languages) { try { // Upload the database bundle. @@ -90,6 +107,7 @@ export async function cleanupAndUploadDatabases( actionsUtil.getRequiredInput("checkout_path"), ); try { + const startTime = performance.now(); await client.request( `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, { @@ -107,13 +125,27 @@ export async function cleanupAndUploadDatabases( }, }, ); + const endTime = performance.now(); + reports.push({ + language, + zipped_upload_size_bytes: bundledDbSize, + is_overlay_base: shouldUploadOverlayBase, + upload_duration_ms: endTime - startTime, + }); logger.debug(`Successfully uploaded database for ${language}`); } finally { bundledDbReadStream.close(); } } catch (e) { // Log a warning but don't fail the workflow - logger.warning(`Failed to upload database for ${language}: ${e}`); + logger.warning( + `Failed to upload database for ${language}: ${util.getErrorMessage(e)}`, + ); + reports.push({ + language, + error: util.getErrorMessage(e), + }); } } + return reports; }