diff --git a/.github/actions/check-codescanning-config/index.ts b/.github/actions/check-codescanning-config/index.ts index 0596e4fe9b..ea99ca3653 100644 --- a/.github/actions/check-codescanning-config/index.ts +++ b/.github/actions/check-codescanning-config/index.ts @@ -6,6 +6,16 @@ import * as assert from 'assert' const actualConfig = loadActualConfig() +function sortConfigArrays(config) { + for (const key of Object.keys(config)) { + const value = config[key]; + if (key === 'queries' && Array.isArray(value)) { + config[key] = value.sort(); + } + } + return config; +} + const rawExpectedConfig = process.argv[3].trim() if (!rawExpectedConfig) { core.setFailed('No expected configuration provided') @@ -18,8 +28,8 @@ if (!rawExpectedConfig) { const expectedConfig = rawExpectedConfig ? JSON.parse(rawExpectedConfig) : undefined; assert.deepStrictEqual( - actualConfig, - expectedConfig, + sortConfigArrays(actualConfig), + sortConfigArrays(expectedConfig), 'Expected configuration does not match actual configuration' ); diff --git a/.github/workflows/codescanning-config-cli.yml b/.github/workflows/codescanning-config-cli.yml index 131c914dd7..316cb7d13c 100644 --- a/.github/workflows/codescanning-config-cli.yml +++ b/.github/workflows/codescanning-config-cli.yml @@ -180,13 +180,13 @@ jobs: with: expected-config-file-contents: | { - "queries": [ - { "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" }, - { "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" } - ], "packs": { "javascript": ["codeql-testing/codeql-pack1@1.0.0", "codeql-testing/codeql-pack2", "codeql/javascript-queries" ] - } + }, + "queries": [ + { "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }, + { "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" } + ] } languages: javascript queries: + ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 927d8150b5..db641a89a9 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -26447,7 +26447,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -117922,6 +117922,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 0424733363..ec0f083cf8 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -32296,7 +32296,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -91156,6 +91156,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index e03abc4131..1b022d3788 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -26447,7 +26447,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -78660,6 +78660,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/init-action-post.js b/lib/init-action-post.js index b0f821950b..83a0f07f2b 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -32296,7 +32296,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -129255,6 +129255,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/init-action.js b/lib/init-action.js index bef6ecd486..0bc5e10eb7 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -32296,7 +32296,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -86128,6 +86128,12 @@ function computeAutomationID(analysis_key, environment) { } return automationID; } +async function getRepositoryProperties(repositoryNwo) { + return getApiClient().request("GET /repos/:owner/:repo/properties/values", { + owner: repositoryNwo.owner, + repo: repositoryNwo.repo + }); +} // src/caching-utils.ts var core6 = __toESM(require_core()); @@ -86223,6 +86229,9 @@ function getConfigFileFormatInvalidMessage(configFile) { function getConfigFileDirectoryGivenMessage(configFile) { return `The configuration file "${configFile}" looks like a directory, not a file`; } +function getEmptyCombinesError() { + return `A '+' was used to specify that you want to add extra arguments to the configuration, but no extra arguments were specified. Please either remove the '+' or specify some extra arguments.`; +} function getConfigFilePropertyError(configFile, property, error2) { if (configFile === void 0) { return `The workflow property "${property}" is invalid: ${error2}`; @@ -86230,6 +86239,9 @@ function getConfigFilePropertyError(configFile, property, error2) { return `The configuration file "${configFile}" is invalid: property "${property}" ${error2}`; } } +function getRepoPropertyError(propertyName, error2) { + return `The repository property "${propertyName}" is invalid: ${error2}`; +} function getPacksStrInvalid(packStr, configFile) { return configFile ? getConfigFilePropertyError( configFile, @@ -86244,6 +86256,52 @@ function getUnknownLanguagesError(languages) { return `Did not recognize the following languages: ${languages.join(", ")}`; } +// src/feature-flags/properties.ts +var RepositoryPropertyName = /* @__PURE__ */ ((RepositoryPropertyName2) => { + RepositoryPropertyName2["EXTRA_QUERIES"] = "github-codeql-extra-queries"; + return RepositoryPropertyName2; +})(RepositoryPropertyName || {}); +async function loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo) { + if (gitHubVersion.type === 1 /* GHES */) { + return {}; + } + try { + const response = await getRepositoryProperties(repositoryNwo); + const remoteProperties = response.data; + if (!Array.isArray(remoteProperties)) { + throw new Error( + `Expected repository properties API to return an array, but got: ${JSON.stringify(response.data)}` + ); + } + logger.debug( + `Retrieved ${remoteProperties.length} repository properties: ${remoteProperties.map((p) => p.property_name).join(", ")}` + ); + const knownProperties = new Set(Object.values(RepositoryPropertyName)); + const properties = {}; + for (const property of remoteProperties) { + if (property.property_name === void 0) { + throw new Error( + `Expected property object to have a 'property_name', but got: ${JSON.stringify(property)}` + ); + } + if (knownProperties.has(property.property_name)) { + properties[property.property_name] = property.value; + } + } + logger.debug("Loaded the following values for the repository properties:"); + for (const [property, value] of Object.entries(properties).sort( + ([nameA], [nameB]) => nameA.localeCompare(nameB) + )) { + logger.debug(` ${property}: ${value}`); + } + return properties; + } catch (e) { + throw new Error( + `Encountered an error while trying to determine repository properties: ${e}` + ); + } +} + // src/config/db-config.ts function shouldCombine(inputValue) { return !!inputValue?.trim().startsWith("+"); @@ -86336,7 +86394,7 @@ function parsePacksFromInput(rawPacksInput, languages, packsInputCombines) { }, []) }; } -async function calculateAugmentation(rawPacksInput, rawQueriesInput, languages) { +async function calculateAugmentation(rawPacksInput, rawQueriesInput, repositoryProperties, languages) { const packsInputCombines = shouldCombine(rawPacksInput); const packsInput = parsePacksFromInput( rawPacksInput, @@ -86348,19 +86406,38 @@ async function calculateAugmentation(rawPacksInput, rawQueriesInput, languages) rawQueriesInput, queriesInputCombines ); + const repoExtraQueries = repositoryProperties["github-codeql-extra-queries" /* EXTRA_QUERIES */]; + const repoExtraQueriesCombines = shouldCombine(repoExtraQueries); + const repoPropertyQueries = { + combines: repoExtraQueriesCombines, + input: parseQueriesFromInput( + repoExtraQueries, + repoExtraQueriesCombines, + new ConfigurationError( + getRepoPropertyError( + "github-codeql-extra-queries" /* EXTRA_QUERIES */, + getEmptyCombinesError() + ) + ) + ) + }; return { packsInputCombines, packsInput: packsInput?.[languages[0]], queriesInput, - queriesInputCombines + queriesInputCombines, + repoPropertyQueries }; } -function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) { +function parseQueriesFromInput(rawQueriesInput, queriesInputCombines, errorToThrow) { if (!rawQueriesInput) { return void 0; } const trimmedInput = queriesInputCombines ? rawQueriesInput.trim().slice(1).trim() : rawQueriesInput?.trim() ?? ""; if (queriesInputCombines && trimmedInput.length === 0) { + if (errorToThrow) { + throw errorToThrow; + } throw new ConfigurationError( getConfigFilePropertyError( void 0, @@ -86371,17 +86448,43 @@ function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) { } return trimmedInput.split(",").map((query) => ({ uses: query.trim() })); } -function generateCodeScanningConfig(originalUserInput, augmentationProperties) { - const augmentedConfig = cloneObject(originalUserInput); - if (augmentationProperties.queriesInput) { - if (augmentationProperties.queriesInputCombines) { - augmentedConfig.queries = (augmentedConfig.queries || []).concat( - augmentationProperties.queriesInput +function combineQueries(logger, config, augmentationProperties) { + const result = []; + if (augmentationProperties.repoPropertyQueries && augmentationProperties.repoPropertyQueries.input) { + logger.info( + `Found query configuration in the repository properties (${"github-codeql-extra-queries" /* EXTRA_QUERIES */}): ${augmentationProperties.repoPropertyQueries.input.map((q) => q.uses).join(", ")}` + ); + if (!augmentationProperties.repoPropertyQueries.combines) { + logger.info( + `The queries configured in the repository properties don't allow combining with other query settings. Any queries configured elsewhere will be ignored.` ); + return augmentationProperties.repoPropertyQueries.input; } else { - augmentedConfig.queries = augmentationProperties.queriesInput; + result.push(...augmentationProperties.repoPropertyQueries.input); } } + if (augmentationProperties.queriesInput) { + if (!augmentationProperties.queriesInputCombines) { + return result.concat(augmentationProperties.queriesInput); + } else { + result.push(...augmentationProperties.queriesInput); + } + } + if (config.queries) { + result.push(...config.queries); + } + return result; +} +function generateCodeScanningConfig(logger, originalUserInput, augmentationProperties) { + const augmentedConfig = cloneObject(originalUserInput); + augmentedConfig.queries = combineQueries( + logger, + augmentedConfig, + augmentationProperties + ); + logger.debug( + `Combined queries: ${augmentedConfig.queries?.map((q) => q.uses).join(",")}` + ); if (augmentedConfig.queries?.length === 0) { delete augmentedConfig.queries; } @@ -86953,6 +87056,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", @@ -87493,6 +87601,7 @@ async function initActionState({ sourceRoot, githubVersion, features, + repositoryProperties, logger }, userConfig) { const analysisKinds = await parseAnalysisKinds(analysisKindsInput); @@ -87516,8 +87625,18 @@ async function initActionState({ const augmentationProperties = await calculateAugmentation( packsInput, queriesInput, + repositoryProperties, languages ); + if (analysisKinds.length === 1 && analysisKinds.includes("code-quality" /* CodeQuality */) && augmentationProperties.repoPropertyQueries.input) { + logger.info( + `Ignoring queries configured in the repository properties, because query customisations are not supported for Code Quality analyses.` + ); + augmentationProperties.repoPropertyQueries = { + combines: false, + input: void 0 + }; + } const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime( trapCachingEnabled, codeql, @@ -87525,6 +87644,7 @@ async function initActionState({ logger ); const computedConfig = generateCodeScanningConfig( + logger, userConfig, augmentationProperties ); @@ -87547,7 +87667,8 @@ async function initActionState({ dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled), extraQueryExclusions: [], overlayDatabaseMode: "none" /* None */, - useOverlayDatabaseCaching: false + useOverlayDatabaseCaching: false, + repositoryProperties }; } async function downloadCacheWithTime(trapCachingEnabled, codeQL, languages, logger) { @@ -90384,6 +90505,10 @@ async function run() { getTemporaryDirectory(), logger ); + const enableRepoProps = await features.getValue( + "use_repository_properties" /* UseRepositoryProperties */ + ); + const repositoryProperties = enableRepoProps ? await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo) : {}; const jobRunUuid = v4_default(); logger.info(`Job run UUID is ${jobRunUuid}.`); core13.exportVariable("JOB_RUN_UUID" /* JOB_RUN_UUID */, jobRunUuid); @@ -90483,6 +90608,7 @@ async function run() { githubVersion: gitHubVersion, apiDetails, features, + repositoryProperties, logger }); await checkInstallPython311(config.languages, codeql); diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index 731fac08e9..7306a6f0d6 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -26447,7 +26447,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -78651,6 +78651,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 90304233b0..5234d9f125 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -26447,7 +26447,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -117331,6 +117331,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/start-proxy-action.js b/lib/start-proxy-action.js index 474b6d820d..7a0475a3a1 100644 --- a/lib/start-proxy-action.js +++ b/lib/start-proxy-action.js @@ -44975,7 +44975,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 4c57a9ddbe..49cec9332b 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -33593,7 +33593,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -89347,6 +89347,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index 0a92128f73..5737173053 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -26447,7 +26447,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -117494,6 +117494,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index a1c04da31a..0b63c40c5d 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -32296,7 +32296,7 @@ var require_package = __commonJS({ lint: "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - test: "npm run transpile && ava src/**.test.ts --serial --verbose", + test: "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", transpile: "tsc --build --verbose" }, @@ -89335,6 +89335,11 @@ var featureConfig = { minimumVersion: void 0, toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */ }, + ["use_repository_properties" /* UseRepositoryProperties */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: void 0 + }, ["qa_telemetry_enabled" /* QaTelemetryEnabled */]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/package.json b/package.json index 8cffcaa509..8c33d89879 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "eslint --report-unused-disable-directives --max-warnings=0 .", "lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix", - "test": "npm run transpile && ava src/**.test.ts --serial --verbose", + "test": "npm run transpile && ava src/ --serial --verbose", "test-debug": "npm run test -- --timeout=20m", "transpile": "tsc --build --verbose" }, diff --git a/src/api-client.ts b/src/api-client.ts index 207b3c86af..8e4a30c571 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -4,7 +4,7 @@ import * as retry from "@octokit/plugin-retry"; import consoleLogLevel from "console-log-level"; import { getActionVersion, getRequiredInput } from "./actions-util"; -import { getRepositoryNwo } from "./repository"; +import { getRepositoryNwo, RepositoryNwo } from "./repository"; import { ConfigurationError, getRequiredEnvParam, @@ -240,6 +240,14 @@ export async function deleteActionsCache(id: number) { }); } +/** Retrieve all custom repository properties. */ +export async function getRepositoryProperties(repositoryNwo: RepositoryNwo) { + return getApiClient().request("GET /repos/:owner/:repo/properties/values", { + owner: repositoryNwo.owner, + repo: repositoryNwo.repo, + }); +} + export function wrapApiConfigurationError(e: unknown) { if (isHTTPError(e)) { if ( diff --git a/src/codeql.test.ts b/src/codeql.test.ts index 36775f6530..a5422b1e38 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -496,6 +496,8 @@ const injectedConfigMacro = test.macro({ expectedConfig: any, ) => { await util.withTmpDir(async (tempDir) => { + sinon.stub(actionsUtil, "isDefaultSetup").resolves(false); + const runnerConstructorStub = stubToolRunnerConstructor(); const codeqlObject = await stubCodeql(); @@ -505,6 +507,7 @@ const injectedConfigMacro = test.macro({ tempDir, }; thisStubConfig.computedConfig = generateCodeScanningConfig( + getRunnerLogger(true), thisStubConfig.originalUserInput, augmentationProperties, ); @@ -659,15 +662,15 @@ test( }, { queries: [ - { - uses: "zzz", - }, { uses: "xxx", }, { uses: "yyy", }, + { + uses: "zzz", + }, ], }, ); @@ -713,6 +716,84 @@ test( {}, ); +test( + "repo property queries have the highest precedence", + injectedConfigMacro, + { + ...defaultAugmentationProperties, + queriesInputCombines: true, + queriesInput: [{ uses: "xxx" }, { uses: "yyy" }], + repoPropertyQueries: { + combines: false, + input: [{ uses: "zzz" }, { uses: "aaa" }], + }, + }, + { + originalUserInput: { + queries: [{ uses: "uu" }, { uses: "vv" }], + }, + }, + { + queries: [{ uses: "zzz" }, { uses: "aaa" }], + }, +); + +test( + "repo property queries combines with queries input", + injectedConfigMacro, + { + ...defaultAugmentationProperties, + queriesInputCombines: false, + queriesInput: [{ uses: "xxx" }, { uses: "yyy" }], + repoPropertyQueries: { + combines: true, + input: [{ uses: "zzz" }, { uses: "aaa" }], + }, + }, + { + originalUserInput: { + queries: [{ uses: "uu" }, { uses: "vv" }], + }, + }, + { + queries: [ + { uses: "zzz" }, + { uses: "aaa" }, + { uses: "xxx" }, + { uses: "yyy" }, + ], + }, +); + +test( + "repo property queries combines everything else", + injectedConfigMacro, + { + ...defaultAugmentationProperties, + queriesInputCombines: true, + queriesInput: [{ uses: "xxx" }, { uses: "yyy" }], + repoPropertyQueries: { + combines: true, + input: [{ uses: "zzz" }, { uses: "aaa" }], + }, + }, + { + originalUserInput: { + queries: [{ uses: "uu" }, { uses: "vv" }], + }, + }, + { + queries: [ + { uses: "zzz" }, + { uses: "aaa" }, + { uses: "xxx" }, + { uses: "yyy" }, + { uses: "uu" }, + { uses: "vv" }, + ], + }, +); + test("passes a code scanning config AND qlconfig to the CLI", async (t: ExecutionContext) => { await util.withTmpDir(async (tempDir) => { const runnerConstructorStub = stubToolRunnerConstructor(); diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index 8938adf845..566a719ca0 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -29,6 +29,7 @@ import { getRecordingLogger, LoggedMessage, mockCodeQLVersion, + createTestConfig, } from "./testing-utils"; import { GitHubVariant, @@ -82,11 +83,11 @@ function createTestInitConfigInputs( externalRepoAuth: "token", url: "https://github.example.com", apiURL: undefined, - registriesAuthTokens: undefined, }, features: createFeatures([]), + repositoryProperties: {}, logger: getRunnerLogger(true), - }, + } satisfies configUtils.InitConfigInputs, overrides, ); } @@ -223,12 +224,70 @@ test("load code quality config", async (t) => { extraQueryExclusions: [], overlayDatabaseMode: OverlayDatabaseMode.None, useOverlayDatabaseCaching: false, + repositoryProperties: {}, }; t.deepEqual(config, expectedConfig); }); }); +test("initActionState doesn't throw if there are queries configured in the repository properties", async (t) => { + return await withTmpDir(async (tempDir) => { + const logger = getRunnerLogger(true); + const languages = "javascript"; + + const codeql = createStubCodeQL({ + async betterResolveLanguages() { + return { + extractors: { + javascript: [{ extractor_root: "" }], + }, + }; + }, + }); + + // This should be ignored and no error should be thrown. + const repositoryProperties = { + "github-codeql-extra-queries": "+foo", + }; + + // Expected configuration for a CQ-only analysis. + const computedConfig: configUtils.UserConfig = { + "disable-default-queries": true, + queries: [{ uses: "code-quality" }], + "query-filters": [], + }; + + const expectedConfig = createTestConfig({ + analysisKinds: [AnalysisKind.CodeQuality], + languages: [KnownLanguage.javascript], + codeQLCmd: codeql.getPath(), + computedConfig, + dbLocation: path.resolve(tempDir, "codeql_databases"), + debugArtifactName: "", + debugDatabaseName: "", + tempDir, + repositoryProperties, + }); + + await t.notThrowsAsync(async () => { + const config = await configUtils.initConfig( + createTestInitConfigInputs({ + analysisKindsInput: "code-quality", + languagesInput: languages, + repository: { owner: "github", repo: "example" }, + tempDir, + codeql, + repositoryProperties, + logger, + }), + ); + + t.deepEqual(config, expectedConfig); + }); + }); +}); + test("loading a saved config produces the same config", async (t) => { return await withTmpDir(async (tempDir) => { const logger = getRunnerLogger(true); @@ -461,6 +520,7 @@ test("load non-empty input", async (t) => { extraQueryExclusions: [], overlayDatabaseMode: OverlayDatabaseMode.None, useOverlayDatabaseCaching: false, + repositoryProperties: {}, }; const languagesInput = "javascript"; diff --git a/src/config-utils.ts b/src/config-utils.ts index 1915102899..fe4b392ab2 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -25,6 +25,7 @@ import { import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils"; import * as errorMessages from "./error-messages"; import { Feature, FeatureEnablement } from "./feature-flags"; +import { RepositoryProperties } from "./feature-flags/properties"; import { getGitRoot, isAnalyzingDefaultBranch } from "./git-utils"; import { KnownLanguage, Language } from "./languages"; import { Logger } from "./logging"; @@ -167,6 +168,11 @@ export interface Config { * `OverlayBase`. */ useOverlayDatabaseCaching: boolean; + + /** + * A partial mapping from repository properties that affect us to their values. + */ + repositoryProperties: RepositoryProperties; } export async function getSupportedLanguageMap( @@ -389,6 +395,7 @@ export interface InitConfigInputs { githubVersion: GitHubVersion; apiDetails: api.GitHubApiCombinedDetails; features: FeatureEnablement; + repositoryProperties: RepositoryProperties; logger: Logger; } @@ -416,6 +423,7 @@ export async function initActionState( sourceRoot, githubVersion, features, + repositoryProperties, logger, }: InitConfigInputs, userConfig: UserConfig, @@ -451,9 +459,28 @@ export async function initActionState( const augmentationProperties = await calculateAugmentation( packsInput, queriesInput, + repositoryProperties, languages, ); + // If `code-quality` is the only enabled analysis kind, we don't support query customisation. + // It would be a problem if queries that are configured in repository properties cause `code-quality`-only + // analyses to break. We therefore ignore query customisations that are configured in repository properties + // if `code-quality` is the only enabled analysis kind. + if ( + analysisKinds.length === 1 && + analysisKinds.includes(AnalysisKind.CodeQuality) && + augmentationProperties.repoPropertyQueries.input + ) { + logger.info( + `Ignoring queries configured in the repository properties, because query customisations are not supported for Code Quality analyses.`, + ); + augmentationProperties.repoPropertyQueries = { + combines: false, + input: undefined, + }; + } + const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime( trapCachingEnabled, codeql, @@ -464,6 +491,7 @@ export async function initActionState( // Compute the full Code Scanning configuration that combines the configuration from the // configuration file / `config` input with other inputs, such as `queries`. const computedConfig = generateCodeScanningConfig( + logger, userConfig, augmentationProperties, ); @@ -488,6 +516,7 @@ export async function initActionState( extraQueryExclusions: [], overlayDatabaseMode: OverlayDatabaseMode.None, useOverlayDatabaseCaching: false, + repositoryProperties, }; } diff --git a/src/config/db-config.test.ts b/src/config/db-config.test.ts index 78750a29ad..b22503475d 100644 --- a/src/config/db-config.test.ts +++ b/src/config/db-config.test.ts @@ -1,5 +1,6 @@ import test, { ExecutionContext } from "ava"; +import { RepositoryProperties } from "../feature-flags/properties"; import { KnownLanguage, Language } from "../languages"; import { prettyPrintPack } from "../util"; @@ -190,11 +191,13 @@ const calculateAugmentationMacro = test.macro({ rawPacksInput: string | undefined, rawQueriesInput: string | undefined, languages: Language[], + repositoryProperties: RepositoryProperties, expectedAugmentationProperties: dbConfig.AugmentationProperties, ) => { const actualAugmentationProperties = await dbConfig.calculateAugmentation( rawPacksInput, rawQueriesInput, + repositoryProperties, languages, ); t.deepEqual(actualAugmentationProperties, expectedAugmentationProperties); @@ -208,6 +211,7 @@ test( undefined, undefined, [KnownLanguage.javascript], + {}, { ...dbConfig.defaultAugmentationProperties, }, @@ -219,6 +223,7 @@ test( undefined, " a, b , c, d", [KnownLanguage.javascript], + {}, { ...dbConfig.defaultAugmentationProperties, queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }], @@ -231,6 +236,7 @@ test( undefined, " + a, b , c, d ", [KnownLanguage.javascript], + {}, { ...dbConfig.defaultAugmentationProperties, queriesInputCombines: true, @@ -244,6 +250,7 @@ test( " codeql/a , codeql/b , codeql/c , codeql/d ", undefined, [KnownLanguage.javascript], + {}, { ...dbConfig.defaultAugmentationProperties, packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"], @@ -256,6 +263,7 @@ test( " + codeql/a, codeql/b, codeql/c, codeql/d", undefined, [KnownLanguage.javascript], + {}, { ...dbConfig.defaultAugmentationProperties, packsInputCombines: true, @@ -263,6 +271,42 @@ test( }, ); +test( + calculateAugmentationMacro, + "With repo property queries", + undefined, + undefined, + [KnownLanguage.javascript], + { + "github-codeql-extra-queries": "a, b, c, d", + }, + { + ...dbConfig.defaultAugmentationProperties, + repoPropertyQueries: { + combines: false, + input: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }], + }, + }, +); + +test( + calculateAugmentationMacro, + "With repo property queries combining", + undefined, + undefined, + [KnownLanguage.javascript], + { + "github-codeql-extra-queries": "+ a, b, c, d", + }, + { + ...dbConfig.defaultAugmentationProperties, + repoPropertyQueries: { + combines: true, + input: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }], + }, + }, +); + const calculateAugmentationErrorMacro = test.macro({ exec: async ( t: ExecutionContext, @@ -270,6 +314,7 @@ const calculateAugmentationErrorMacro = test.macro({ rawPacksInput: string | undefined, rawQueriesInput: string | undefined, languages: Language[], + repositoryProperties: RepositoryProperties, expectedError: RegExp | string, ) => { await t.throwsAsync( @@ -277,6 +322,7 @@ const calculateAugmentationErrorMacro = test.macro({ dbConfig.calculateAugmentation( rawPacksInput, rawQueriesInput, + repositoryProperties, languages, ), { message: expectedError }, @@ -291,6 +337,7 @@ test( undefined, " + ", [KnownLanguage.javascript], + {}, /The workflow property "queries" is invalid/, ); @@ -300,15 +347,29 @@ test( " + ", undefined, [KnownLanguage.javascript], + {}, /The workflow property "packs" is invalid/, ); +test( + calculateAugmentationErrorMacro, + "Plus (+) with nothing else (repo property queries)", + undefined, + undefined, + [KnownLanguage.javascript], + { + "github-codeql-extra-queries": " + ", + }, + /The repository property "github-codeql-extra-queries" is invalid/, +); + test( calculateAugmentationErrorMacro, "Packs input with multiple languages", " + a/b, c/d ", undefined, [KnownLanguage.javascript, KnownLanguage.java], + {}, /Cannot specify a 'packs' input in a multi-language analysis/, ); @@ -318,6 +379,7 @@ test( " + a/b, c/d ", undefined, [], + {}, /No languages specified/, ); @@ -327,5 +389,6 @@ test( " a-pack-without-a-scope ", undefined, [KnownLanguage.javascript], + {}, /"a-pack-without-a-scope" is not a valid pack/, ); diff --git a/src/config/db-config.ts b/src/config/db-config.ts index 95138460f8..2639493543 100644 --- a/src/config/db-config.ts +++ b/src/config/db-config.ts @@ -3,7 +3,12 @@ import * as path from "path"; import * as semver from "semver"; import * as errorMessages from "../error-messages"; +import { + RepositoryProperties, + RepositoryPropertyName, +} from "../feature-flags/properties"; import { Language } from "../languages"; +import { Logger } from "../logging"; import { cloneObject, ConfigurationError, prettyPrintPack } from "../util"; export interface ExcludeQueryFilter { @@ -16,16 +21,18 @@ export interface IncludeQueryFilter { export type QueryFilter = ExcludeQueryFilter | IncludeQueryFilter; +export interface QuerySpec { + name?: string; + uses: string; +} + /** * Format of the config file supplied by the user. */ export interface UserConfig { name?: string; "disable-default-queries"?: boolean; - queries?: Array<{ - name?: string; - uses: string; - }>; + queries?: QuerySpec[]; "paths-ignore"?: string[]; paths?: string[]; @@ -39,6 +46,17 @@ export interface UserConfig { "query-filters"?: QueryFilter[]; } +/** + * Represents additional configuration data from a source other than + * a configuration file. + */ +interface Augmentation { + /** Whether or not the `input` combines with data in the base config. */ + combines: boolean; + /** The additional input data. */ + input?: T; +} + /** * Describes how to augment the user config with inputs from the action. * @@ -58,7 +76,7 @@ export interface AugmentationProperties { /** * The queries input from the `with` block of the action declaration */ - queriesInput?: Array<{ uses: string }>; + queriesInput?: QuerySpec[]; /** * Whether or not the packs input combines with the packs in the config. @@ -69,6 +87,11 @@ export interface AugmentationProperties { * The packs input from the `with` block of the action declaration */ packsInput?: string[]; + + /** + * Extra queries from the corresponding repository property. + */ + repoPropertyQueries: Augmentation; } /** @@ -80,6 +103,10 @@ export const defaultAugmentationProperties: AugmentationProperties = { packsInputCombines: false, packsInput: undefined, queriesInput: undefined, + repoPropertyQueries: { + combines: false, + input: undefined, + }, }; /** @@ -254,6 +281,7 @@ export function parsePacksFromInput( * * @param rawPacksInput The packs input from the action configuration. * @param rawQueriesInput The queries input from the action configuration. + * @param repositoryProperties The dictionary of repository properties. * @param languages The languages that the config file is for. If the packs input * is non-empty, then there must be exactly one language. Otherwise, an * error is thrown. @@ -263,10 +291,10 @@ export function parsePacksFromInput( * @throws An error if the packs input is non-empty and the languages input does * not have exactly one language. */ -// exported for testing. export async function calculateAugmentation( rawPacksInput: string | undefined, rawQueriesInput: string | undefined, + repositoryProperties: RepositoryProperties, languages: Language[], ): Promise { const packsInputCombines = shouldCombine(rawPacksInput); @@ -281,17 +309,36 @@ export async function calculateAugmentation( queriesInputCombines, ); + const repoExtraQueries = + repositoryProperties[RepositoryPropertyName.EXTRA_QUERIES]; + const repoExtraQueriesCombines = shouldCombine(repoExtraQueries); + const repoPropertyQueries = { + combines: repoExtraQueriesCombines, + input: parseQueriesFromInput( + repoExtraQueries, + repoExtraQueriesCombines, + new ConfigurationError( + errorMessages.getRepoPropertyError( + RepositoryPropertyName.EXTRA_QUERIES, + errorMessages.getEmptyCombinesError(), + ), + ), + ), + }; + return { packsInputCombines, packsInput: packsInput?.[languages[0]], queriesInput, queriesInputCombines, + repoPropertyQueries, }; } function parseQueriesFromInput( rawQueriesInput: string | undefined, queriesInputCombines: boolean, + errorToThrow?: ConfigurationError, ) { if (!rawQueriesInput) { return undefined; @@ -301,6 +348,9 @@ function parseQueriesFromInput( ? rawQueriesInput.trim().slice(1).trim() : (rawQueriesInput?.trim() ?? ""); if (queriesInputCombines && trimmedInput.length === 0) { + if (errorToThrow) { + throw errorToThrow; + } throw new ConfigurationError( errorMessages.getConfigFilePropertyError( undefined, @@ -312,7 +362,71 @@ function parseQueriesFromInput( return trimmedInput.split(",").map((query) => ({ uses: query.trim() })); } +/** + * Combines queries from various configuration sources. + * + * @param logger The logger to use. + * @param config The loaded configuration file (either `config-file` or `config` input). + * @param augmentationProperties Additional configuration data from other sources. + * @returns Returns `augmentedConfig` with `queries` set to the computed array of queries. + */ +function combineQueries( + logger: Logger, + config: UserConfig, + augmentationProperties: AugmentationProperties, +): QuerySpec[] { + const result: QuerySpec[] = []; + + // Query settings obtained from the repository properties have the highest precedence. + if ( + augmentationProperties.repoPropertyQueries && + augmentationProperties.repoPropertyQueries.input + ) { + logger.info( + `Found query configuration in the repository properties (${RepositoryPropertyName.EXTRA_QUERIES}): ` + + `${augmentationProperties.repoPropertyQueries.input.map((q) => q.uses).join(", ")}`, + ); + + // If there are queries configured as a repository property, these may be organisational + // settings. If they don't allow combining with other query configurations, return just the + // ones configured in the repository properties. + if (!augmentationProperties.repoPropertyQueries.combines) { + logger.info( + `The queries configured in the repository properties don't allow combining with other query settings. ` + + `Any queries configured elsewhere will be ignored.`, + ); + return augmentationProperties.repoPropertyQueries.input; + } else { + // Otherwise, add them to the query array and continue. + result.push(...augmentationProperties.repoPropertyQueries.input); + } + } + + // If there is a `queries` input to the Action, it has the next highest precedence. + if (augmentationProperties.queriesInput) { + // If there is a `queries` input and `queriesInputCombines` is `false`, then we don't + // combine it with the queries configured in the configuration file (if any). That is the + // original behaviour of this property. However, we DO combine it with any queries that + // we obtained from the repository properties, since that may be enforced by the organisation. + if (!augmentationProperties.queriesInputCombines) { + return result.concat(augmentationProperties.queriesInput); + } else { + // If they combine, add them to the query array and continue. + result.push(...augmentationProperties.queriesInput); + } + } + + // If we get to this point, we either don't have any extra configuration inputs or all of them + // allow themselves to be combined with the settings from the configuration file. + if (config.queries) { + result.push(...config.queries); + } + + return result; +} + export function generateCodeScanningConfig( + logger: Logger, originalUserInput: UserConfig, augmentationProperties: AugmentationProperties, ): UserConfig { @@ -320,15 +434,14 @@ export function generateCodeScanningConfig( const augmentedConfig = cloneObject(originalUserInput); // Inject the queries from the input - if (augmentationProperties.queriesInput) { - if (augmentationProperties.queriesInputCombines) { - augmentedConfig.queries = (augmentedConfig.queries || []).concat( - augmentationProperties.queriesInput, - ); - } else { - augmentedConfig.queries = augmentationProperties.queriesInput; - } - } + augmentedConfig.queries = combineQueries( + logger, + augmentedConfig, + augmentationProperties, + ); + logger.debug( + `Combined queries: ${augmentedConfig.queries?.map((q) => q.uses).join(",")}`, + ); if (augmentedConfig.queries?.length === 0) { delete augmentedConfig.queries; } diff --git a/src/error-messages.ts b/src/error-messages.ts index 61dd3ef92e..eb49266771 100644 --- a/src/error-messages.ts +++ b/src/error-messages.ts @@ -1,3 +1,5 @@ +import { RepositoryPropertyName } from "./feature-flags/properties"; + const PACKS_PROPERTY = "packs"; export function getConfigFileOutsideWorkspaceErrorMessage( @@ -29,6 +31,10 @@ export function getConfigFileDirectoryGivenMessage(configFile: string): string { return `The configuration file "${configFile}" looks like a directory, not a file`; } +export function getEmptyCombinesError(): string { + return `A '+' was used to specify that you want to add extra arguments to the configuration, but no extra arguments were specified. Please either remove the '+' or specify some extra arguments.`; +} + export function getConfigFilePropertyError( configFile: string | undefined, property: string, @@ -41,6 +47,13 @@ export function getConfigFilePropertyError( } } +export function getRepoPropertyError( + propertyName: RepositoryPropertyName, + error: string, +): string { + return `The repository property "${propertyName}" is invalid: ${error}`; +} + export function getPacksStrInvalid( packStr: string, configFile?: string, diff --git a/src/feature-flags.ts b/src/feature-flags.ts index b7946d62f4..2938f5108c 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -73,6 +73,7 @@ export enum Feature { OverlayAnalysisRust = "overlay_analysis_rust", OverlayAnalysisSwift = "overlay_analysis_swift", PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib", + UseRepositoryProperties = "use_repository_properties", QaTelemetryEnabled = "qa_telemetry_enabled", ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli", } @@ -264,6 +265,11 @@ export const featureConfig: Record< minimumVersion: undefined, toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib, }, + [Feature.UseRepositoryProperties]: { + defaultValue: false, + envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES", + minimumVersion: undefined, + }, [Feature.QaTelemetryEnabled]: { defaultValue: false, envVar: "CODEQL_ACTION_QA_TELEMETRY", diff --git a/src/feature-flags/properties.test.ts b/src/feature-flags/properties.test.ts new file mode 100644 index 0000000000..dd0c72a21e --- /dev/null +++ b/src/feature-flags/properties.test.ts @@ -0,0 +1,97 @@ +import test from "ava"; +import * as sinon from "sinon"; + +import * as api from "../api-client"; +import { getRunnerLogger } from "../logging"; +import { parseRepositoryNwo } from "../repository"; +import { setupTests } from "../testing-utils"; +import * as util from "../util"; + +import * as properties from "./properties"; + +setupTests(test); + +test("loadPropertiesFromApi throws if response data is not an array", async (t) => { + sinon.stub(api, "getRepositoryProperties").resolves({ + headers: {}, + status: 200, + url: "", + data: {}, + }); + const logger = getRunnerLogger(true); + const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); + await t.throwsAsync( + properties.loadPropertiesFromApi( + { + type: util.GitHubVariant.DOTCOM, + }, + logger, + mockRepositoryNwo, + ), + ); +}); + +test("loadPropertiesFromApi throws if response data contains unexpected objects", async (t) => { + sinon.stub(api, "getRepositoryProperties").resolves({ + headers: {}, + status: 200, + url: "", + data: [{}], + }); + const logger = getRunnerLogger(true); + const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); + await t.throwsAsync( + properties.loadPropertiesFromApi( + { + type: util.GitHubVariant.DOTCOM, + }, + logger, + mockRepositoryNwo, + ), + ); +}); + +test("loadPropertiesFromApi returns empty object if on GHES", async (t) => { + sinon.stub(api, "getRepositoryProperties").resolves({ + headers: {}, + status: 200, + url: "", + data: [ + { property_name: "github-codeql-extra-queries", value: "+queries" }, + { property_name: "unknown-property", value: "something" }, + ] satisfies properties.RepositoryProperty[], + }); + const logger = getRunnerLogger(true); + const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); + const response = await properties.loadPropertiesFromApi( + { + type: util.GitHubVariant.GHES, + version: "", + }, + logger, + mockRepositoryNwo, + ); + t.deepEqual(response, {}); +}); + +test("loadPropertiesFromApi loads known properties", async (t) => { + sinon.stub(api, "getRepositoryProperties").resolves({ + headers: {}, + status: 200, + url: "", + data: [ + { property_name: "github-codeql-extra-queries", value: "+queries" }, + { property_name: "unknown-property", value: "something" }, + ] satisfies properties.RepositoryProperty[], + }); + const logger = getRunnerLogger(true); + const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); + const response = await properties.loadPropertiesFromApi( + { + type: util.GitHubVariant.DOTCOM, + }, + logger, + mockRepositoryNwo, + ); + t.deepEqual(response, { "github-codeql-extra-queries": "+queries" }); +}); diff --git a/src/feature-flags/properties.ts b/src/feature-flags/properties.ts new file mode 100644 index 0000000000..0104cddd91 --- /dev/null +++ b/src/feature-flags/properties.ts @@ -0,0 +1,94 @@ +import { getRepositoryProperties } from "../api-client"; +import { Logger } from "../logging"; +import { RepositoryNwo } from "../repository"; +import { GitHubVariant, GitHubVersion } from "../util"; + +/** + * Enumerates repository property names that have some meaning to us. + */ +export enum RepositoryPropertyName { + EXTRA_QUERIES = "github-codeql-extra-queries", +} + +/** + * A repository property has a name and a value. + */ +export interface RepositoryProperty { + property_name: string; + value: string; +} + +/** + * The API returns a list of `RepositoryProperty` objects. + */ +type GitHubPropertiesResponse = RepositoryProperty[]; + +/** + * A partial mapping from `RepositoryPropertyName` to values. + */ +export type RepositoryProperties = Partial< + Record +>; + +/** + * Retrieves all known repository properties from the API. + * + * @param logger The logger to use. + * @param repositoryNwo Information about the repository for which to load properties. + * @returns Returns a partial mapping from `RepositoryPropertyName` to values. + */ +export async function loadPropertiesFromApi( + gitHubVersion: GitHubVersion, + logger: Logger, + repositoryNwo: RepositoryNwo, +): Promise { + // TODO: To be safe for now; later we should replace this with a version check once we know + // which version of GHES we expect this to be supported by. + if (gitHubVersion.type === GitHubVariant.GHES) { + return {}; + } + + try { + const response = await getRepositoryProperties(repositoryNwo); + const remoteProperties = response.data as GitHubPropertiesResponse; + + if (!Array.isArray(remoteProperties)) { + throw new Error( + `Expected repository properties API to return an array, but got: ${JSON.stringify(response.data)}`, + ); + } + + logger.debug( + `Retrieved ${remoteProperties.length} repository properties: ${remoteProperties.map((p) => p.property_name).join(", ")}`, + ); + + const knownProperties = new Set(Object.values(RepositoryPropertyName)); + const properties: RepositoryProperties = {}; + for (const property of remoteProperties) { + if (property.property_name === undefined) { + throw new Error( + `Expected property object to have a 'property_name', but got: ${JSON.stringify(property)}`, + ); + } + + if ( + knownProperties.has(property.property_name as RepositoryPropertyName) + ) { + properties[property.property_name] = property.value; + } + } + + logger.debug("Loaded the following values for the repository properties:"); + for (const [property, value] of Object.entries(properties).sort( + ([nameA], [nameB]) => nameA.localeCompare(nameB), + )) { + logger.debug(` ${property}: ${value}`); + } + + return properties; + } catch (e) { + throw new Error( + `Encountered an error while trying to determine repository properties: ${e}`, + ); + } +} diff --git a/src/init-action.ts b/src/init-action.ts index 508d17333b..2b4dba3fcf 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -32,6 +32,7 @@ import { } from "./diagnostics"; import { EnvVar } from "./environment"; import { Feature, Features } from "./feature-flags"; +import { loadPropertiesFromApi } from "./feature-flags/properties"; import { checkInstallPython311, checkPacksForOverlayCompatibility, @@ -196,6 +197,14 @@ async function run() { logger, ); + // Fetch the values of known repository properties that affect us. + const enableRepoProps = await features.getValue( + Feature.UseRepositoryProperties, + ); + const repositoryProperties = enableRepoProps + ? await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo) + : {}; + const jobRunUuid = uuidV4(); logger.info(`Job run UUID is ${jobRunUuid}.`); core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid); @@ -317,6 +326,7 @@ async function run() { githubVersion: gitHubVersion, apiDetails, features, + repositoryProperties, logger, }); diff --git a/src/testing-utils.ts b/src/testing-utils.ts index c930d5350c..ea3929131c 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -378,6 +378,7 @@ export function createTestConfig(overrides: Partial): Config { extraQueryExclusions: [], overlayDatabaseMode: OverlayDatabaseMode.None, useOverlayDatabaseCaching: false, + repositoryProperties: {}, } satisfies Config, overrides, );