diff --git a/.chronus/changes/python-addBackApiViewSphinx-2026-3-9-12-35-1.md b/.chronus/changes/python-addBackApiViewSphinx-2026-3-9-12-35-1.md new file mode 100644 index 00000000000..746bd59aabb --- /dev/null +++ b/.chronus/changes/python-addBackApiViewSphinx-2026-3-9-12-35-1.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Add apiview and sphinx to ci \ No newline at end of file diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index fdcfeeef10c..49110135355 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -42,21 +42,24 @@ function addDefaultOptions(sdkContext: PythonSdkContext) { const packageName = namespace.replace(/\./g, "-"); options["package-name"] = packageName; } - if ((options as any).flavor !== "azure") { - // if they pass in a flavor other than azure, we want to ignore the value - (options as any).flavor = undefined; - } + // Set flavor based on namespace or passed option if (getRootNamespace(sdkContext).toLowerCase().includes("azure")) { (options as any).flavor = "azure"; + } else if ((options as any).flavor !== "azure") { + // Explicitly set unbranded flavor when not azure + (options as any).flavor = "unbranded"; } if ( options["package-pprint-name"] !== undefined && !options["package-pprint-name"].startsWith('"') ) { - options["package-pprint-name"] = options["use-pyodide"] - ? `${options["package-pprint-name"]}` - : `"${options["package-pprint-name"]}"`; + // Only add quotes for shell compatibility when NOT using emit-yaml-only mode + // (emit-yaml-only passes options via JSON config files, not shell) + const needsShellQuoting = !options["use-pyodide"] && !options["emit-yaml-only"]; + options["package-pprint-name"] = needsShellQuoting + ? `"${options["package-pprint-name"]}"` + : `${options["package-pprint-name"]}`; } } @@ -246,6 +249,21 @@ async function onEmitMain(context: EmitContext) { const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap); if (!program.compilerOptions.noEmit && !program.hasError()) { + // If emit-yaml-only mode, just copy YAML to output dir for batch processing + if (resolvedOptions["emit-yaml-only"]) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + // Copy YAML to output dir with command args embedded + // Use unique filename to avoid conflicts when multiple specs share output dir + const configId = path.basename(yamlPath, ".yaml"); + const batchConfig = { yamlPath, commandArgs, outputDir }; + fs.writeFileSync( + path.join(outputDir, `.tsp-codegen-${configId}.json`), + JSON.stringify(batchConfig, null, 2), + ); + return; + } // if not using pyodide and there's no venv, we try to create venv if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) { try { diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index 2d6010108e0..b267c836bf3 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -25,6 +25,7 @@ export interface PythonEmitterOptions { "use-pyodide"?: boolean; "keep-setup-py"?: boolean; "clear-output-folder"?: boolean; + "emit-yaml-only"?: boolean; } export interface PythonSdkContext extends SdkContext { @@ -110,6 +111,12 @@ export const PythonEmitterOptionsSchema: JSONSchemaType = description: "Whether to clear the output folder before generating the code. Defaults to `false`.", }, + "emit-yaml-only": { + type: "boolean", + nullable: true, + description: + "Emit YAML code model only, without running Python generator. For batch processing.", + }, }, required: [], }; diff --git a/packages/http-client-python/eng/scripts/ci/config/eslint-ci.config.mjs b/packages/http-client-python/eng/scripts/ci/config/eslint-ci.config.mjs index b8a1418a63e..d630457015f 100644 --- a/packages/http-client-python/eng/scripts/ci/config/eslint-ci.config.mjs +++ b/packages/http-client-python/eng/scripts/ci/config/eslint-ci.config.mjs @@ -40,6 +40,7 @@ export default [ "no-case-declarations": "off", "no-ex-assign": "off", "no-undef": "off", + "no-useless-assignment": "error", "prefer-const": [ "warn", { diff --git a/packages/http-client-python/eng/scripts/ci/lint.ts b/packages/http-client-python/eng/scripts/ci/lint.ts index 9d6150b9acc..9dad879a992 100644 --- a/packages/http-client-python/eng/scripts/ci/lint.ts +++ b/packages/http-client-python/eng/scripts/ci/lint.ts @@ -133,14 +133,21 @@ function runCommand( } async function lintEmitter(): Promise { - console.log(`\n${pc.bold("=== Linting TypeScript Emitter ===")}\n`); + console.log(`\n${pc.bold("=== Linting TypeScript ===")}\n`); // Run eslint with local config to avoid dependency on monorepo's eslint.config.js // This ensures the package can be linted in CI without full monorepo dependencies + // Lint both emitter/ and eng/scripts/ directories return runCommand( "eslint", - ["emitter/", "--config", "eng/scripts/ci/config/eslint-ci.config.mjs", "--max-warnings=0"], + [ + "emitter/", + "eng/scripts/", + "--config", + "eng/scripts/ci/config/eslint-ci.config.mjs", + "--max-warnings=0", + ], root, - "eslint emitter/ --max-warnings=0", + "eslint emitter/ eng/scripts/ --max-warnings=0", ); } diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index d49f5ac4d71..145ba9b21b1 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -7,7 +7,8 @@ */ import { compile, NodeHost } from "@typespec/compiler"; -import { rmSync } from "fs"; +import { execSync } from "child_process"; +import { existsSync, readdirSync, rmSync } from "fs"; import { platform } from "os"; import { dirname, join, relative, resolve } from "path"; import pc from "picocolors"; @@ -173,6 +174,9 @@ function buildTaskGroups(specs: string[], flags: RegenerateFlags): TaskGroup[] { // Examples directory options["examples-dir"] = toPosix(join(dirname(spec), "examples")); + // Emit YAML only - Python processing is batched after all specs compile + options["emit-yaml-only"] = true; + tasks.push({ spec, outputDir, options }); } @@ -213,6 +217,25 @@ async function compileSpec(task: CompileTask): Promise<{ success: boolean; error } } +function renderProgressBar( + completed: number, + failed: number, + total: number, + width: number = 40, +): string { + const successCount = completed - failed; + const successWidth = Math.round((successCount / total) * width); + const failWidth = Math.round((failed / total) * width); + const emptyWidth = width - successWidth - failWidth; + + const successBar = pc.bgGreen(" ".repeat(successWidth)); + const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; + const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); + + const percent = Math.round((completed / total) * 100); + return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; +} + async function runParallel(groups: TaskGroup[], maxJobs: number): Promise> { const results = new Map(); const executing: Set> = new Set(); @@ -220,6 +243,20 @@ async function runParallel(groups: TaskGroup[], maxJobs: number): Promise sum + g.tasks.length, 0); let completed = 0; + let failed = 0; + const failedSpecs: string[] = []; + + // Check if we're in a TTY for progress bar updates + const isTTY = process.stdout.isTTY; + + const updateProgress = () => { + if (isTTY) { + process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); + } + }; + + // Initial progress bar + updateProgress(); for (const group of groups) { // Each group runs as a unit - tasks within a group run sequentially @@ -232,19 +269,17 @@ async function runParallel(groups: TaskGroup[], maxJobs: number): Promise 0) { + console.log(pc.red(`\nFailed specs:`)); + for (const spec of failedSpecs) { + console.log(pc.red(` • ${spec}`)); + } + } + return results; } +function collectConfigFiles(generatedDir: string, flavor: string): string[] { + const flavorDir = join(generatedDir, "..", "tests", "generated", flavor); + if (!existsSync(flavorDir)) return []; + + const configFiles: string[] = []; + for (const pkg of readdirSync(flavorDir, { withFileTypes: true })) { + if (pkg.isDirectory()) { + const pkgDir = join(flavorDir, pkg.name); + // Find all .tsp-codegen-*.json files (supports multiple configs per output dir) + for (const file of readdirSync(pkgDir)) { + if (file.startsWith(".tsp-codegen-") && file.endsWith(".json")) { + configFiles.push(join(pkgDir, file)); + } + } + } + } + return configFiles; +} + +function runBatchPythonProcessing(flavor: string, configCount: number, jobs: number): boolean { + if (configCount === 0) return true; + + console.log(pc.cyan(`\nRunning batch Python processing on ${configCount} specs...`)); + + // Find Python venv + let venvPath = join(PLUGIN_DIR, "venv"); + if (existsSync(join(venvPath, "bin"))) { + venvPath = join(venvPath, "bin", "python"); + } else if (existsSync(join(venvPath, "Scripts"))) { + venvPath = join(venvPath, "Scripts", "python.exe"); + } else { + console.error(pc.red("Python venv not found")); + return false; + } + + const batchScript = join(PLUGIN_DIR, "eng", "scripts", "setup", "run_batch.py"); + + try { + // Pass directory and flavor instead of individual config files to avoid command line length limits on Windows + execSync( + `"${venvPath}" "${batchScript}" --generated-dir "${PLUGIN_DIR}" --flavor ${flavor} --jobs ${jobs}`, + { + stdio: "inherit", + cwd: PLUGIN_DIR, + }, + ); + return true; + } catch { + return false; + } +} + async function regenerateFlavor( flavor: string, name: string | undefined, @@ -289,21 +390,46 @@ async function regenerateFlavor( console.log(pc.cyan(`Found ${allSpecs.length} specs (${totalTasks} total tasks) to compile`)); console.log(pc.cyan(`Using ${jobs} parallel jobs\n`)); - // Run compilation + // Run compilation (emits YAML only) const startTime = performance.now(); const results = await runParallel(groups, jobs); - const duration = (performance.now() - startTime) / 1000; + const compileTime = (performance.now() - startTime) / 1000; - // Summary + // Summary for TypeSpec compilation const succeeded = Array.from(results.values()).filter((v) => v).length; - const failed = results.size - succeeded; + const compileFailed = results.size - succeeded; + + console.log( + pc.cyan( + `\nTypeSpec compilation: ${succeeded} succeeded, ${compileFailed} failed (${compileTime.toFixed(1)}s)`, + ), + ); + + if (compileFailed > 0) { + console.log(pc.red(`Skipping Python processing due to compilation failures`)); + return false; + } + + // Batch process all specs with Python + const pyStartTime = performance.now(); + const configCount = collectConfigFiles(GENERATED_FOLDER, flavor).length; + // Use fewer Python jobs since Python processing is heavier + const pyJobs = Math.max(4, Math.floor(jobs / 2)); + const pySuccess = runBatchPythonProcessing(flavor, configCount, pyJobs); + const pyTime = (performance.now() - pyStartTime) / 1000; + + const totalTime = (performance.now() - startTime) / 1000; console.log(pc.cyan(`\n${"=".repeat(60)}`)); - console.log(pc.cyan(`Results: ${succeeded} succeeded, ${failed} failed`)); - console.log(pc.cyan(`Time: ${duration.toFixed(1)}s`)); + console.log(pc.cyan(`Results: ${succeeded} specs processed`)); + console.log( + pc.cyan( + ` TypeSpec: ${compileTime.toFixed(1)}s | Python: ${pyTime.toFixed(1)}s | Total: ${totalTime.toFixed(1)}s`, + ), + ); console.log(pc.cyan(`${"=".repeat(60)}\n`)); - return failed === 0; + return pySuccess; } async function main() { diff --git a/packages/http-client-python/eng/scripts/ci/run-tests.ts b/packages/http-client-python/eng/scripts/ci/run-tests.ts index c5409113dff..8cbb80488a1 100644 --- a/packages/http-client-python/eng/scripts/ci/run-tests.ts +++ b/packages/http-client-python/eng/scripts/ci/run-tests.ts @@ -80,7 +80,13 @@ interface ToxResult { error?: string; } -async function runToxEnv(env: string, pythonPath: string, name?: string): Promise { +interface RunningTask { + env: string; + proc: ChildProcess; + promise: Promise; +} + +function startToxEnv(env: string, pythonPath: string, name?: string): RunningTask { const startTime = Date.now(); const toxIniPath = join(testsDir, "tox.ini"); @@ -92,20 +98,20 @@ async function runToxEnv(env: string, pythonPath: string, name?: string): Promis console.log(`${pc.blue("[START]")} ${env}`); - return new Promise((resolve) => { - const proc: ChildProcess = spawn(pythonPath, args, { - cwd: testsDir, - stdio: !argv.values.quiet ? "inherit" : "pipe", - env: { ...process.env, FOLDER: env.split("-")[1] || "azure" }, - }); + const proc: ChildProcess = spawn(pythonPath, args, { + cwd: testsDir, + stdio: !argv.values.quiet ? "inherit" : "pipe", + env: { ...process.env, FOLDER: env.split("-")[1] || "azure" }, + }); - let stderr = ""; - if (argv.values.quiet && proc.stderr) { - proc.stderr.on("data", (data) => { - stderr += data.toString(); - }); - } + let stderr = ""; + if (argv.values.quiet && proc.stderr) { + proc.stderr.on("data", (data) => { + stderr += data.toString(); + }); + } + const promise = new Promise((resolve) => { proc.on("close", (code) => { const duration = (Date.now() - startTime) / 1000; const success = code === 0; @@ -135,6 +141,16 @@ async function runToxEnv(env: string, pythonPath: string, name?: string): Promis }); }); }); + + return { env, proc, promise }; +} + +function killTask(task: RunningTask): void { + try { + task.proc.kill("SIGTERM"); + } catch { + // Process may have already exited + } } async function runParallel( @@ -144,24 +160,51 @@ async function runParallel( name?: string, ): Promise { const results: ToxResult[] = []; - const running: Map> = new Map(); + const running: Map = new Map(); for (const env of envs) { // Wait if we're at max capacity - if (running.size >= maxJobs) { - const completed = await Promise.race(running.values()); + while (running.size >= maxJobs) { + const promises = Array.from(running.values()).map((t) => t.promise); + const completed = await Promise.race(promises); results.push(completed); running.delete(completed.env); + + // Fail-fast: kill all running tasks and exit + if (!completed.success) { + console.log(pc.red(`\n[FAIL-FAST] ${completed.env} failed, killing remaining tasks...`)); + for (const task of running.values()) { + killTask(task); + } + // Wait briefly for processes to terminate + await Promise.all(Array.from(running.values()).map((t) => t.promise)); + return results; + } } // Start new task - const task = runToxEnv(env, pythonPath, name); + const task = startToxEnv(env, pythonPath, name); running.set(env, task); } - // Wait for remaining tasks - const remaining = await Promise.all(running.values()); - results.push(...remaining); + // Wait for remaining tasks, checking for failures + while (running.size > 0) { + const promises = Array.from(running.values()).map((t) => t.promise); + const completed = await Promise.race(promises); + results.push(completed); + running.delete(completed.env); + + // Fail-fast: kill all running tasks and exit + if (!completed.success) { + console.log(pc.red(`\n[FAIL-FAST] ${completed.env} failed, killing remaining tasks...`)); + for (const task of running.values()) { + killTask(task); + } + // Wait briefly for processes to terminate + await Promise.all(Array.from(running.values()).map((t) => t.promise)); + return results; + } + } return results; } @@ -348,37 +391,22 @@ async function main(): Promise { process.exit(1); } - // Separate test environments from other environments - // Test environments must run sequentially (they share port 3000) - // Other environments (lint, mypy, pyright, docs) can run in parallel - const testEnvs = envs.filter((e) => e.startsWith("test-") || e === "unittest"); - const otherEnvs = envs.filter((e) => !e.startsWith("test-") && e !== "unittest"); - const maxJobs = argv.values.jobs ? parseInt(argv.values.jobs, 10) : Math.max(2, cpus().length - 2); console.log(` Flavors: ${flavors.join(", ")}`); console.log(` Environments: ${envs.join(", ")}`); - console.log(` Jobs: ${maxJobs} (test envs run sequentially, others in parallel)`); + console.log(` Jobs: ${maxJobs}`); if (argv.values.name) { console.log(` Filter: ${argv.values.name}`); } console.log(); - // Run test environments first (sequentially) - let results: ToxResult[] = []; - if (testEnvs.length > 0) { - console.log(pc.cyan("Running test environments (sequential)...")); - results = await runParallel(testEnvs, pythonPath, 1, argv.values.name); - } - - // Run other environments in parallel - if (otherEnvs.length > 0) { - console.log(pc.cyan("\nRunning lint/typecheck environments (parallel)...")); - const otherResults = await runParallel(otherEnvs, pythonPath, maxJobs, argv.values.name); - results = results.concat(otherResults); - } + // Run all environments in parallel + // The mock server serves both azure and unbranded specs, so all tests can run together + console.log(pc.cyan("Running all environments in parallel...")); + const results = await runParallel(envs, pythonPath, maxJobs, argv.values.name); allResults.push(...results); } diff --git a/packages/http-client-python/eng/scripts/ci/run_apiview.py b/packages/http-client-python/eng/scripts/ci/run_apiview.py index 5345f694ef0..d69cca7f4d6 100644 --- a/packages/http-client-python/eng/scripts/ci/run_apiview.py +++ b/packages/http-client-python/eng/scripts/ci/run_apiview.py @@ -10,32 +10,38 @@ import os import sys -from subprocess import check_call, CalledProcessError +from subprocess import run, TimeoutExpired import logging from util import run_check logging.getLogger().setLevel(logging.INFO) +# Timeout for each apiview generation (seconds) +APIVIEW_TIMEOUT = 60 + def _single_dir_apiview(mod): - loop = 0 - while True: + for attempt in range(3): try: - check_call( - [ - "apistubgen", - "--pkg-path", - str(mod.absolute()), - ] + result = run( + ["apistubgen", "--pkg-path", str(mod.absolute())], + capture_output=True, + timeout=APIVIEW_TIMEOUT, ) - except CalledProcessError as e: - if loop >= 2: # retry for maximum 3 times because sometimes the apistubgen has transient failure. - logging.error("{} exited with apiview generation error {}".format(mod.stem, e.returncode)) + if result.returncode == 0: + return True + if attempt == 2: + logging.error(f"{mod.stem} failed: {result.stderr.decode()[:200]}") + return False + except TimeoutExpired: + if attempt == 2: + logging.error(f"{mod.stem} timed out after {APIVIEW_TIMEOUT}s") + return False + except Exception as e: + if attempt == 2: + logging.error(f"{mod.stem} error: {e}") return False - else: - loop += 1 - continue - return True + return False if __name__ == "__main__": diff --git a/packages/http-client-python/eng/scripts/ci/run_sphinx_build.py b/packages/http-client-python/eng/scripts/ci/run_sphinx_build.py index e97476aed9e..143fa1707a7 100644 --- a/packages/http-client-python/eng/scripts/ci/run_sphinx_build.py +++ b/packages/http-client-python/eng/scripts/ci/run_sphinx_build.py @@ -8,7 +8,7 @@ # This script is used to execute sphinx documentation build within a tox environment. # It uses a central sphinx configuration and validates docstrings by running sphinx-build. -from subprocess import check_call, CalledProcessError +from subprocess import run, TimeoutExpired import os import logging import sys @@ -20,6 +20,9 @@ # Get the central Sphinx config directory SPHINX_CONF_DIR = os.path.abspath(os.path.dirname(__file__)) +# Timeout for each sphinx build (seconds) +SPHINX_TIMEOUT = 120 + def _create_minimal_index_rst(docs_dir, package_name, module_names): """Create a minimal index.rst file for sphinx to process.""" @@ -85,7 +88,7 @@ def _single_dir_sphinx(mod): sys.path.insert(0, str(mod.absolute())) try: - result = check_call( + result = run( [ sys.executable, "-m", @@ -100,12 +103,19 @@ def _single_dir_sphinx(mod): "-q", # Quiet mode (only show warnings/errors) str(docs_dir.absolute()), # Source directory str(output_dir.absolute()), # Output directory - ] + ], + capture_output=True, + timeout=SPHINX_TIMEOUT, ) - logging.info(f"Sphinx build completed successfully for {mod.stem}") - return True - except CalledProcessError as e: - logging.error(f"{mod.stem} exited with sphinx build error {e.returncode}") + if result.returncode == 0: + return True + logging.error(f"{mod.stem} sphinx error: {result.stderr.decode()[:500]}") + return False + except TimeoutExpired: + logging.error(f"{mod.stem} timed out after {SPHINX_TIMEOUT}s") + return False + except Exception as e: + logging.error(f"{mod.stem} sphinx error: {e}") return False finally: # Remove from sys.path diff --git a/packages/http-client-python/eng/scripts/ci/util.py b/packages/http-client-python/eng/scripts/ci/util.py index 63a51c4d754..3822a300ddf 100644 --- a/packages/http-client-python/eng/scripts/ci/util.py +++ b/packages/http-client-python/eng/scripts/ci/util.py @@ -8,10 +8,11 @@ import logging from pathlib import Path import argparse -from multiprocessing import Pool +from concurrent.futures import ProcessPoolExecutor, as_completed logging.getLogger().setLevel(logging.INFO) +# Root is the tests directory (4 levels up from this file: ci -> scripts -> eng -> package_root, then into tests) ROOT_FOLDER = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", "..", "tests")) IGNORE_FOLDER = [] @@ -25,16 +26,9 @@ def run_check(name, call_back, log_info): "-t", "--test-folder", dest="test_folder", - help="The test folder we're in. Can be 'azure' or 'vanilla'", + help="The test folder we're in. Can be 'azure' or 'unbranded'", required=True, ) - parser.add_argument( - "-g", - "--generator", - dest="generator", - help="The generator we're using. Can be 'legacy', 'version-tolerant'.", - required=False, - ) parser.add_argument( "-f", "--file-name", @@ -46,28 +40,52 @@ def run_check(name, call_back, log_info): "-s", "--subfolder", dest="subfolder", - help="The specific sub folder to validate, default to Expected/AcceptanceTests. Optional.", + help="The subfolder containing generated code, default to 'generated'.", + required=False, + default="generated", + ) + parser.add_argument( + "-j", + "--jobs", + dest="jobs", + help="Number of parallel jobs (default: CPU count / 2)", + type=int, required=False, - default="Expected/AcceptanceTests", + default=max(1, os.cpu_count() // 2), ) args = parser.parse_args() - pkg_dir = Path(ROOT_FOLDER) - if args.subfolder: - pkg_dir /= Path(args.subfolder) - pkg_dir /= Path(args.test_folder) - if args.generator: - pkg_dir /= Path(args.generator) + # Path structure: tests/generated/{test_folder}/ + pkg_dir = Path(ROOT_FOLDER) / Path(args.subfolder) / Path(args.test_folder) dirs = [d for d in pkg_dir.iterdir() if d.is_dir() and not d.stem.startswith("_") and d.stem not in IGNORE_FOLDER] if args.file_name: dirs = [d for d in dirs if args.file_name.lower() in d.stem.lower()] - if len(dirs) > 1: - with Pool() as pool: - result = pool.map(call_back, dirs) - response = all(result) - else: - response = call_back(dirs[0]) - if not response: - logging.error("%s fails", log_info) + + if not dirs: + logging.info("No directories to process") + return + + logging.info(f"Processing {len(dirs)} packages with {args.jobs} parallel jobs...") + + failed = [] + succeeded = 0 + + with ProcessPoolExecutor(max_workers=args.jobs) as executor: + futures = {executor.submit(call_back, d): d for d in dirs} + for future in as_completed(futures): + pkg = futures[future] + try: + if future.result(): + succeeded += 1 + else: + failed.append(pkg.stem) + except Exception as e: + logging.error(f"{pkg.stem} raised exception: {e}") + failed.append(pkg.stem) + + logging.info(f"{log_info}: {succeeded} succeeded, {len(failed)} failed") + + if failed: + logging.error(f"{log_info} failed for: {', '.join(failed)}") exit(1) diff --git a/packages/http-client-python/eng/scripts/setup/run_batch.py b/packages/http-client-python/eng/scripts/setup/run_batch.py new file mode 100644 index 00000000000..55b8ce39cf2 --- /dev/null +++ b/packages/http-client-python/eng/scripts/setup/run_batch.py @@ -0,0 +1,177 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +""" +Batch process multiple TypeSpec YAML files in a single Python process. +This avoids the overhead of spawning a new Python process for each spec. +""" +import argparse +import json +import sys +import os +from pathlib import Path +from concurrent.futures import ProcessPoolExecutor, as_completed +from multiprocessing import freeze_support + +# Add the generator to the path +_ROOT_DIR = Path(__file__).parent.parent.parent.parent +sys.path.insert(0, str(_ROOT_DIR / "generator")) + + +def process_single_spec(config_path_str: str) -> tuple[str, bool, str]: + """Process a single spec from its config file. + + Returns: (output_dir, success, error_message) + """ + # Import inside function for multiprocessing compatibility + from pygen import preprocess, codegen, black + + config_path = Path(config_path_str) + try: + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + yaml_path = config["yamlPath"] + command_args = config["commandArgs"] + output_dir = config["outputDir"] + + # Pass command args directly to pygen - pygen expects hyphenated keys + # Remove keys that shouldn't be passed to pygen + pygen_args = {k: v for k, v in command_args.items() if k not in ["emit-yaml-only"]} + + # Run preprocess and codegen + preprocess.PreProcessPlugin(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process() + + codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process() + + # Run black + black.BlackScriptPlugin(output_folder=output_dir, **pygen_args).process() + + # Clean up the config file + config_path.unlink() + + return (output_dir, True, "") + except Exception as e: + return (str(config_path), False, str(e)) + + +def render_progress_bar(completed: int, failed: int, total: int, width: int = 40) -> str: + """Render a progress bar with green for success and red for failures.""" + success_count = completed - failed + success_width = round((success_count / total) * width) if total > 0 else 0 + fail_width = round((failed / total) * width) if total > 0 else 0 + empty_width = width - success_width - fail_width + + # ANSI color codes + green_bg = "\033[42m" + red_bg = "\033[41m" + reset = "\033[0m" + dim = "\033[2m" + cyan = "\033[36m" + + success_bar = f"{green_bg}{' ' * success_width}{reset}" + fail_bar = f"{red_bg}{' ' * fail_width}{reset}" if failed > 0 else "" + empty_bar = f"{dim}{'░' * max(0, empty_width)}{reset}" + + percent = round((completed / total) * 100) if total > 0 else 0 + return f"{success_bar}{fail_bar}{empty_bar} {cyan}{percent}%{reset} ({completed}/{total})" + + +def collect_config_files(generated_dir: str, flavor: str) -> list[str]: + """Collect all .tsp-codegen-*.json config files from the generated directory.""" + flavor_dir = Path(generated_dir) / "tests" / "generated" / flavor + if not flavor_dir.exists(): + return [] + + config_files = [] + for pkg_dir in flavor_dir.iterdir(): + if pkg_dir.is_dir(): + for f in pkg_dir.iterdir(): + if f.name.startswith(".tsp-codegen-") and f.name.endswith(".json"): + config_files.append(str(f)) + return config_files + + +def main(): + parser = argparse.ArgumentParser(description="Batch process TypeSpec YAML files") + parser.add_argument( + "--generated-dir", + required=True, + help="Path to the generator directory (config files are in ../tests/generated//)", + ) + parser.add_argument( + "--flavor", + required=True, + help="Flavor to process (azure or unbranded)", + ) + parser.add_argument( + "--jobs", + type=int, + default=4, + help="Number of parallel jobs (default: 4)", + ) + args = parser.parse_args() + + # Discover config files from the generated directory + config_files = collect_config_files(args.generated_dir, args.flavor) + total = len(config_files) + + if total == 0: + print("No config files found, nothing to process") + return + + print(f"Processing {total} specs with {args.jobs} parallel jobs...") + + succeeded = 0 + failed = 0 + failed_specs = [] + is_tty = sys.stdout.isatty() + + def update_progress(): + if is_tty: + sys.stdout.write(f"\r{render_progress_bar(succeeded + failed, failed, total)}") + sys.stdout.flush() + + # Initial progress bar + update_progress() + + # Use ProcessPoolExecutor for true parallelism (bypasses GIL) + with ProcessPoolExecutor(max_workers=args.jobs) as executor: + futures = {executor.submit(process_single_spec, cf): cf for cf in config_files} + + for future in as_completed(futures): + output_dir, success, error = future.result() + if success: + succeeded += 1 + else: + failed += 1 + failed_specs.append(f"{output_dir}: {error}") + # Fail-fast: cancel pending futures on first failure + print(f"\n\033[31m[FAIL-FAST] Cancelling remaining tasks after failure...\033[0m") + for f in futures: + f.cancel() + break + update_progress() + + # Clear progress bar line + if is_tty: + sys.stdout.write("\r" + " " * 60 + "\r") + sys.stdout.flush() + + # Print failures at the end + if failed_specs: + print("\n\033[31mFailed specs:\033[0m") + for spec in failed_specs: + print(f" \033[31m•\033[0m {spec}") + + print(f"\nBatch processing complete: {succeeded} succeeded, {failed} failed") + + if failed > 0: + sys.exit(1) + + +if __name__ == "__main__": + freeze_support() # Required for Windows multiprocessing + main() diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 018605575c9..7092019f5d0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -242,7 +242,7 @@ def _serialize_and_write_package_files(self) -> None: lstrip_blocks=True, ) - package_files = _PACKAGE_FILES + package_files = list(_PACKAGE_FILES) # Copy to avoid modifying global if not self.code_model.license_description: package_files.remove("LICENSE.jinja2") elif Path(self.code_model.options["package-mode"]).exists(): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py index 54fe489920d..ce907c55427 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py @@ -155,7 +155,10 @@ def serialize_package_file(self, template_name: str, file_content: str, **kwargs "VERSION_MAP": VERSION_MAP, "MIN_PYTHON_VERSION": MIN_PYTHON_VERSION, "MAX_PYTHON_VERSION": MAX_PYTHON_VERSION, - "ADDITIONAL_DEPENDENCIES": [f"{item[0]}>={item[1]}" for item in additional_version_map.items()], + "ADDITIONAL_DEPENDENCIES": [ + dep if dep.startswith('"') else f'"{dep}"' + for dep in (f"{item[0]}>={item[1]}" for item in additional_version_map.items()) + ], } params |= {"options": self.code_model.options} params |= kwargs diff --git a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2 index 5cc41d91cb9..4a012849501 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/packaging_templates/pyproject.toml.jinja2 @@ -57,7 +57,7 @@ dependencies = [ {% endfor %} {% endif %} {% for dep in ADDITIONAL_DEPENDENCIES %} - "{{ dep }}", + {{ dep }}, {% endfor %} ] dynamic = [ diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 1b911555093..644950c2daf 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -46,7 +46,7 @@ "typecheck": "tsx ./eng/scripts/ci/typecheck.ts", "typecheck:generated": "tsx ./eng/scripts/ci/typecheck.ts --generated", "regenerate": "tsx ./eng/scripts/ci/regenerate.ts", - "ci": "npm run test && npm run lint && npm run typecheck", + "ci": "npm run test:emitter && npm run ci:generated", "ci:generated": "tsx ./eng/scripts/ci/run-tests.ts --generator --env=ci", "change:version": "pnpm chronus version --ignore-policies --only @typespec/http-client-python", "change:add": "pnpm chronus add", diff --git a/packages/http-client-python/tests/conftest.py b/packages/http-client-python/tests/conftest.py index d3780e9e216..70222f11ece 100644 --- a/packages/http-client-python/tests/conftest.py +++ b/packages/http-client-python/tests/conftest.py @@ -50,23 +50,20 @@ def wait_for_server(url: str, timeout: int = 60, interval: float = 0.5) -> bool: def start_server_process(): - """Start the tsp-spector mock API server.""" + """Start the tsp-spector mock API server. + + Always serves both azure-http-specs and http-specs regardless of flavor. + This allows azure and unbranded tests to run in parallel using the same server. + """ azure_http_path = ROOT / "node_modules/@azure-tools/azure-http-specs" http_path = ROOT / "node_modules/@typespec/http-specs" - # Determine flavor from environment or current directory - flavor = os.environ.get("FLAVOR", "azure") - + # Always serve both spec sets so azure and unbranded tests can run in parallel # Use absolute paths with forward slashes (works on all platforms including Windows) - if flavor == "unbranded": - cwd = http_path.resolve() - specs_path = str(cwd / "specs").replace("\\", "/") - cmd = f"npx tsp-spector serve {specs_path}" - else: - cwd = azure_http_path.resolve() - azure_specs = str(cwd / "specs").replace("\\", "/") - http_specs = str((http_path / "specs").resolve()).replace("\\", "/") - cmd = f"npx tsp-spector serve {azure_specs} {http_specs}" + cwd = azure_http_path.resolve() + azure_specs = str(cwd / "specs").replace("\\", "/") + http_specs = str((http_path / "specs").resolve()).replace("\\", "/") + cmd = f"npx tsp-spector serve {azure_specs} {http_specs}" # Add node_modules/.bin to PATH env = os.environ.copy() diff --git a/packages/http-client-python/tests/install_packages.py b/packages/http-client-python/tests/install_packages.py index f706ae93732..903d57c62e1 100644 --- a/packages/http-client-python/tests/install_packages.py +++ b/packages/http-client-python/tests/install_packages.py @@ -19,9 +19,20 @@ def install_packages(flavor: str, tests_dir: str) -> None: print(f"Warning: Generated directory does not exist: {generated_dir}") return - # Find all package directories - packages = glob.glob(os.path.join(generated_dir, "*")) - packages = [p for p in packages if os.path.isdir(p)] + # Find all package directories that have pyproject.toml or setup.py + all_dirs = glob.glob(os.path.join(generated_dir, "*")) + packages = [ + p for p in all_dirs + if os.path.isdir(p) and ( + os.path.exists(os.path.join(p, "pyproject.toml")) or + os.path.exists(os.path.join(p, "setup.py")) + ) + ] + + # Log skipped directories for debugging + skipped = [os.path.basename(p) for p in all_dirs if os.path.isdir(p) and p not in packages] + if skipped: + print(f"Skipping {len(skipped)} directories without packaging files: {', '.join(skipped[:5])}{'...' if len(skipped) > 5 else ''}") if not packages: print(f"Warning: No packages found in {generated_dir}") @@ -31,19 +42,37 @@ def install_packages(flavor: str, tests_dir: str) -> None: # Install packages using uv pip # Use --no-deps to avoid dependency resolution overhead - cmd = ["uv", "pip", "install", "--no-deps"] + packages - - try: - subprocess.run(cmd, check=True) - print(f"Successfully installed {len(packages)} packages") - except subprocess.CalledProcessError as e: - print(f"Error installing packages: {e}") - sys.exit(1) - except FileNotFoundError: - # uv not found, try pip - print("uv not found, falling back to pip") - cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + packages - subprocess.run(cmd, check=True) + # Install in batches to avoid command line length limits on Windows + batch_size = 20 # Conservative batch size for Windows command line limits + use_uv = True + + for i in range(0, len(packages), batch_size): + batch = packages[i:i + batch_size] + batch_num = i // batch_size + 1 + total_batches = (len(packages) + batch_size - 1) // batch_size + + if total_batches > 1: + print(f" Batch {batch_num}/{total_batches}: {len(batch)} packages") + + if use_uv: + cmd = ["uv", "pip", "install", "--no-deps"] + batch + else: + cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + batch + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"Error installing packages: {e}") + sys.exit(1) + except FileNotFoundError: + if use_uv: + # uv not found, fall back to pip for this and subsequent batches + print("uv not found, falling back to pip") + use_uv = False + cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + batch + subprocess.run(cmd, check=True) + + print(f"Successfully installed {len(packages)} packages") def main(): diff --git a/packages/http-client-python/tests/tox.ini b/packages/http-client-python/tests/tox.ini index 12231969731..67f75ca53ed 100644 --- a/packages/http-client-python/tests/tox.ini +++ b/packages/http-client-python/tests/tox.ini @@ -28,7 +28,7 @@ deps = -e {tox_root}/../generator commands = python {tox_root}/install_packages.py azure {tox_root} - pytest mock_api/azure mock_api/shared -v -n auto -n auto {posargs} + pytest mock_api/azure mock_api/shared -v -n auto {posargs} [testenv:test-unbranded] description = Run tests for unbranded flavor @@ -41,7 +41,7 @@ deps = -e {tox_root}/../generator commands = python {tox_root}/install_packages.py unbranded {tox_root} - pytest mock_api/unbranded mock_api/shared -v -n auto -n auto {posargs} + pytest mock_api/unbranded mock_api/shared -v -n auto {posargs} [testenv:unittest] description = Run unit tests for pygen internals