diff --git a/extensions/ql-vscode/src/codeql-cli/cli-version.ts b/extensions/ql-vscode/src/codeql-cli/cli-version.ts index 5d3f7305688..ebd30cbdb62 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli-version.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli-version.ts @@ -10,9 +10,6 @@ interface VersionResult { } export interface CliFeatures { - featuresInVersionResult?: boolean; - mrvaPackCreate?: boolean; - generateSummarySymbolMap?: boolean; queryServerRunQueries?: boolean; } diff --git a/extensions/ql-vscode/src/codeql-cli/cli.ts b/extensions/ql-vscode/src/codeql-cli/cli.ts index a34b6a351ea..f32c4a2555b 100644 --- a/extensions/ql-vscode/src/codeql-cli/cli.ts +++ b/extensions/ql-vscode/src/codeql-cli/cli.ts @@ -1191,15 +1191,12 @@ export class CodeQLCliServer implements Disposable { outputPath: string, endSummaryPath: string, ): Promise { - const supportsGenerateSummarySymbolMap = - await this.cliConstraints.supportsGenerateSummarySymbolMap(); const subcommandArgs = [ "--format=text", `--end-summary=${endSummaryPath}`, "--sourcemap", - ...(supportsGenerateSummarySymbolMap - ? ["--summary-symbol-map", "--minify-output"] - : []), + "--summary-symbol-map", + "--minify-output", inputPath, outputPath, ]; @@ -1910,11 +1907,7 @@ export class CliVersionConstraint { /**/ } - async supportsMrvaPackCreate(): Promise { - return (await this.cli.getFeatures()).mrvaPackCreate === true; - } - - async supportsGenerateSummarySymbolMap(): Promise { - return (await this.cli.getFeatures()).generateSummarySymbolMap === true; + async supportsQueryServerRunQueries(): Promise { + return (await this.cli.getFeatures()).queryServerRunQueries === true; } } diff --git a/extensions/ql-vscode/src/log-insights/summary-parser.ts b/extensions/ql-vscode/src/log-insights/summary-parser.ts index f00923e53ea..61a3a8af250 100644 --- a/extensions/ql-vscode/src/log-insights/summary-parser.ts +++ b/extensions/ql-vscode/src/log-insights/summary-parser.ts @@ -1,6 +1,3 @@ -import { createReadStream, writeFile } from "fs-extra"; -import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream"; - /** * Location information for a single pipeline invocation in the RA. */ @@ -32,103 +29,3 @@ interface PredicateSymbol { export interface SummarySymbols { predicates: Record; } - -// Tuple counts for Expr::Expr::getParent#dispred#f0820431#ff@76d6745o: -const NON_RECURSIVE_TUPLE_COUNT_REGEXP = - /^Evaluated relational algebra for predicate (?\S+) with tuple counts:$/; -// Tuple counts for Expr::Expr::getEnclosingStmt#f0820431#bf@923ddwj9 on iteration 0 running pipeline base: -const RECURSIVE_TUPLE_COUNT_REGEXP = - /^Evaluated relational algebra for predicate (?\S+) on iteration (?\d+) running pipeline (?\S+) with tuple counts:$/; -const RETURN_REGEXP = /^\s*return /; - -/** - * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline - * run. - * - * TODO: Once we're more certain about the symbol format, we should have the CLI generate this as it - * generates the human-readabe summary to avoid having to rely on regular expression matching of the - * human-readable text. - * - * @param summaryPath The path to the summary file. - * @param symbolsPath The path to the symbols file to generate. - */ -export async function generateSummarySymbolsFile( - summaryPath: string, - symbolsPath: string, -): Promise { - const symbols = await generateSummarySymbols(summaryPath); - await writeFile(symbolsPath, JSON.stringify(symbols)); -} - -/** - * Parse a human-readable evaluation log summary to find the location of the RA for each pipeline - * run. - * - * @param fileLocation The path to the summary file. - * @returns Symbol information for the summary file. - */ -async function generateSummarySymbols( - summaryPath: string, -): Promise { - const stream = createReadStream(summaryPath, { - encoding: "utf-8", - }); - try { - const lines = splitStreamAtSeparators(stream, LINE_ENDINGS); - - const symbols: SummarySymbols = { - predicates: {}, - }; - - let lineNumber = 0; - let raStartLine = 0; - let iteration = 0; - let predicateName: string | undefined = undefined; - let startLine = 0; - for await (const line of lines) { - if (predicateName === undefined) { - // Looking for the start of the predicate. - const nonRecursiveMatch = line.match(NON_RECURSIVE_TUPLE_COUNT_REGEXP); - if (nonRecursiveMatch) { - iteration = 0; - predicateName = nonRecursiveMatch.groups!.predicateName; - } else { - const recursiveMatch = line.match(RECURSIVE_TUPLE_COUNT_REGEXP); - if (recursiveMatch?.groups) { - predicateName = recursiveMatch.groups.predicateName; - iteration = parseInt(recursiveMatch.groups.iteration); - } - } - if (predicateName !== undefined) { - startLine = lineNumber; - raStartLine = lineNumber + 1; - } - } else { - const returnMatch = line.match(RETURN_REGEXP); - if (returnMatch) { - let symbol = symbols.predicates[predicateName]; - if (symbol === undefined) { - symbol = { - iterations: {}, - recursionSummaries: {}, - }; - symbols.predicates[predicateName] = symbol; - } - symbol.iterations[iteration] = { - startLine, - raStartLine, - raEndLine: lineNumber, - }; - - predicateName = undefined; - } - } - - lineNumber++; - } - - return symbols; - } finally { - stream.close(); - } -} diff --git a/extensions/ql-vscode/src/query-server/query-server-client.ts b/extensions/ql-vscode/src/query-server/query-server-client.ts index d52af580dea..8087686992f 100644 --- a/extensions/ql-vscode/src/query-server/query-server-client.ts +++ b/extensions/ql-vscode/src/query-server/query-server-client.ts @@ -100,7 +100,7 @@ export class QueryServerClient extends DisposableObject { * queries at once. */ async supportsRunQueriesMethod(): Promise { - return (await this.cliServer.getFeatures()).queryServerRunQueries === true; + return await this.cliServer.cliConstraints.supportsQueryServerRunQueries(); } /** Stops the query server by disposing of the current server process. */ diff --git a/extensions/ql-vscode/src/run-queries-shared.ts b/extensions/ql-vscode/src/run-queries-shared.ts index 990d3d8293d..4e463812cb0 100644 --- a/extensions/ql-vscode/src/run-queries-shared.ts +++ b/extensions/ql-vscode/src/run-queries-shared.ts @@ -27,7 +27,6 @@ import type { import type { BaseLogger } from "./common/logging"; import { showAndLogWarningMessage } from "./common/logging"; import { extLogger } from "./common/logging/vscode"; -import { generateSummarySymbolsFile } from "./log-insights/summary-parser"; import { getErrorMessage } from "./common/helpers-pure"; import { createHash } from "crypto"; import { QueryOutputDir } from "./local-queries/query-output-dir"; @@ -570,15 +569,6 @@ export async function generateEvalLogSummaries( if (humanReadableSummary !== undefined) { summarySymbols = outputDir.evalLogSummarySymbolsPath; - if ( - !(await cliServer.cliConstraints.supportsGenerateSummarySymbolMap()) - ) { - // We're using an old CLI that cannot generate the summary symbols file while generating the - // human-readable log summary. As a fallback, create it by parsing the human-readable - // summary. - progress(progressUpdate(3, 3, "Generating summary symbols file")); - await generateSummarySymbolsFile(humanReadableSummary, summarySymbols); - } } } diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts index e7e67a0df6c..5b29eed8fd0 100644 --- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts +++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts @@ -1,7 +1,7 @@ import type { CancellationToken } from "vscode"; -import { Uri, window } from "vscode"; -import { join, sep, basename, relative } from "path"; -import { dump, load } from "js-yaml"; +import { window } from "vscode"; +import { join, basename, relative } from "path"; +import { dump } from "js-yaml"; import { copy, writeFile, readFile, mkdirp, remove } from "fs-extra"; import { nanoid } from "nanoid"; import { tmpDir } from "../tmp-dir"; @@ -27,13 +27,7 @@ import { } from "./repository-selection"; import type { Repository } from "./shared/repository"; import type { DbManager } from "../databases/db-manager"; -import { - getQlPackFilePath, - FALLBACK_QLPACK_FILENAME, - QLPACK_FILENAMES, - QLPACK_LOCK_FILENAMES, -} from "../common/ql"; -import type { QlPackFile } from "../packaging/qlpack-file"; +import { FALLBACK_QLPACK_FILENAME } from "../common/ql"; import { expandShortPaths } from "../common/short-paths"; import type { QlPackDetails } from "./ql-pack-details"; import type { ModelPackDetails } from "../common/model-pack-details"; @@ -68,8 +62,6 @@ async function generateQueryPack( ); const mustSynthesizePack = qlPackDetails.qlPackFilePath === undefined; - const cliSupportsMrvaPackCreate = - await cliServer.cliConstraints.supportsMrvaPackCreate(); let targetPackPath: string; let needsInstall: boolean; @@ -87,15 +79,6 @@ async function generateQueryPack( // Install packs, since we just synthesized a dependency on the language's standard library. needsInstall = true; - } else if (!cliSupportsMrvaPackCreate) { - // We need to copy the query pack to a temporary directory and then fix it up to work with MRVA. - targetPackPath = tmpDir.queryPackDir; - await copyExistingQueryPack(cliServer, qlPackDetails, targetPackPath); - - // We should already have all the dependencies available, but these older versions of the CLI - // have a bug where they will not search `--additional-packs` during validation in `codeql pack bundle`. - // Installing the packs will ensure that any extension packs get put in the right place. - needsInstall = true; } else { // The CLI supports creating a MRVA query pack directly from the source pack. targetPackPath = qlPackDetails.qlPackRootPath; @@ -113,27 +96,18 @@ async function generateQueryPack( await cliServer.clearCache(); } - let precompilationOpts: string[]; - if (cliSupportsMrvaPackCreate) { - const queryOpts = qlPackDetails.queryFiles.flatMap((q) => [ - "--query", - join(targetPackPath, relative(qlPackDetails.qlPackRootPath, q)), - ]); - - precompilationOpts = [ - "--mrva", - ...queryOpts, - // We need to specify the extension packs as dependencies so that they are included in the MRVA pack. - // The version range doesn't matter, since they'll always be found by source lookup. - ...extensionPacks.map((p) => `--extension-pack=${p.name}@*`), - ]; - } else { - precompilationOpts = ["--qlx"]; - - if (extensionPacks.length > 0) { - await addExtensionPacksAsDependencies(targetPackPath, extensionPacks); - } - } + const queryOpts = qlPackDetails.queryFiles.flatMap((q) => [ + "--query", + join(targetPackPath, relative(qlPackDetails.qlPackRootPath, q)), + ]); + + const precompilationOpts = [ + "--mrva", + ...queryOpts, + // We need to specify the extension packs as dependencies so that they are included in the MRVA pack. + // The version range doesn't matter, since they'll always be found by source lookup. + ...extensionPacks.map((p) => `--extension-pack=${p.name}@*`), + ]; const bundlePath = tmpDir.bundleFile; void extLogger.log( @@ -181,59 +155,6 @@ async function createNewQueryPack( ); } -async function copyExistingQueryPack( - cliServer: CodeQLCliServer, - qlPackDetails: QlPackDetails, - targetPackPath: string, -) { - const toCopy = await cliServer.packPacklist( - qlPackDetails.qlPackRootPath, - false, - ); - - // Also include query files that contain extensible predicates. These query files are not - // needed for the query to run, but they are needed for the query pack to pass deep validation - // of data extensions. - const metadata = await cliServer.generateExtensiblePredicateMetadata( - qlPackDetails.qlPackRootPath, - ); - metadata.extensible_predicates.forEach((predicate) => { - if (predicate.path.endsWith(".ql")) { - toCopy.push(join(qlPackDetails.qlPackRootPath, predicate.path)); - } - }); - - [ - // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist. - ...QLPACK_LOCK_FILENAMES.map((f) => join(qlPackDetails.qlPackRootPath, f)), - ...qlPackDetails.queryFiles, - ].forEach((absolutePath) => { - if (absolutePath) { - toCopy.push(absolutePath); - } - }); - - let copiedCount = 0; - await copy(qlPackDetails.qlPackRootPath, targetPackPath, { - filter: (file: string) => - // copy file if it is in the packlist, or it is a parent directory of a file in the packlist - !!toCopy.find((f) => { - // Normalized paths ensure that Windows drive letters are capitalized consistently. - const normalizedPath = Uri.file(f).fsPath; - const matches = - normalizedPath === file || normalizedPath.startsWith(file + sep); - if (matches) { - copiedCount++; - } - return matches; - }), - }); - - void extLogger.log(`Copied ${copiedCount} files to ${targetPackPath}`); - - await fixPackFile(targetPackPath, qlPackDetails); -} - interface RemoteQueryTempDir { remoteQueryDir: string; queryPackDir: string; @@ -351,42 +272,6 @@ export async function prepareRemoteQueryRun( }; } -/** - * Fixes the qlpack.yml or codeql-pack.yml file to be correct in the context of the MRVA request. - * - * Performs the following fixes: - * - * - Updates the default suite of the query pack. This is used to ensure - * only the specified query is run. - * - Ensures the query pack name is set to the name expected by the server. - * - Removes any `${workspace}` version references from the qlpack.yml or codeql-pack.yml file. Converts them - * to `*` versions. - * - * @param targetPackPath The path to the directory containing the target pack - * @param qlPackDetails The details of the original QL pack - */ -async function fixPackFile( - targetPackPath: string, - qlPackDetails: QlPackDetails, -): Promise { - const packPath = await getQlPackFilePath(targetPackPath); - - // This should not happen since we create the pack ourselves. - if (!packPath) { - throw new Error( - `Could not find ${QLPACK_FILENAMES.join( - " or ", - )} file in '${targetPackPath}'`, - ); - } - const qlpack = load(await readFile(packPath, "utf8")) as QlPackFile; - - updateDefaultSuite(qlpack, qlPackDetails); - removeWorkspaceRefs(qlpack); - - await writeFile(packPath, dump(qlpack)); -} - async function getExtensionPacksToInject( cliServer: CodeQLCliServer, workspaceFolders: string[], @@ -415,41 +300,6 @@ async function getExtensionPacksToInject( return result; } -async function addExtensionPacksAsDependencies( - queryPackDir: string, - extensionPacks: ModelPackDetails[], -): Promise { - const qlpackFile = await getQlPackFilePath(queryPackDir); - if (!qlpackFile) { - throw new Error( - `Could not find ${QLPACK_FILENAMES.join( - " or ", - )} file in '${queryPackDir}'`, - ); - } - - const syntheticQueryPack = load( - await readFile(qlpackFile, "utf8"), - ) as QlPackFile; - - const dependencies = syntheticQueryPack.dependencies ?? {}; - extensionPacks.forEach(({ name }) => { - // Add this extension pack as a dependency. It doesn't matter which - // version we specify, since we are guaranteed that the extension pack - // is resolved from source at the given path. - dependencies[name] = "*"; - }); - - syntheticQueryPack.dependencies = dependencies; - - await writeFile(qlpackFile, dump(syntheticQueryPack)); -} - -function updateDefaultSuite(qlpack: QlPackFile, qlPackDetails: QlPackDetails) { - delete qlpack.defaultSuiteFile; - qlpack.defaultSuite = generateDefaultSuite(qlPackDetails); -} - function generateDefaultSuite(qlPackDetails: QlPackDetails) { const queries = qlPackDetails.queryFiles.map((query) => { const relativePath = relative(qlPackDetails.qlPackRootPath, query); @@ -550,15 +400,3 @@ async function getControllerRepoFromApi( } } } - -function removeWorkspaceRefs(qlpack: QlPackFile) { - if (!qlpack.dependencies) { - return; - } - - for (const [key, value] of Object.entries(qlpack.dependencies)) { - if (value === "${workspace}") { - qlpack.dependencies[key] = "*"; - } - } -}