|
| 1 | +import * as fs from 'fs'; |
| 2 | +import * as path from 'path'; |
| 3 | +import { join } from 'path'; |
| 4 | + |
| 5 | +export interface TestFailure { |
| 6 | + testName: string; |
| 7 | + type: 'empty-scip-index' | 'missing-output' | 'content-mismatch' | 'orphaned-output'; |
| 8 | + message: string; |
| 9 | +} |
| 10 | + |
| 11 | +export interface TestError { |
| 12 | + testName: string; |
| 13 | + type: 'invalid-filter' | 'empty-scip-index'; |
| 14 | + message: string; |
| 15 | +} |
| 16 | + |
| 17 | +export interface ValidationResults { |
| 18 | + passed: string[]; |
| 19 | + failed: TestFailure[]; |
| 20 | + errors: TestError[]; |
| 21 | + skipped: string[]; |
| 22 | +} |
| 23 | + |
| 24 | +export interface TestRunnerOptions { |
| 25 | + snapshotRoot: string; |
| 26 | + filterTests?: string; |
| 27 | + failFast?: boolean; |
| 28 | + quiet?: boolean; |
| 29 | +} |
| 30 | + |
| 31 | +export interface SingleTestOptions { |
| 32 | + check: boolean; |
| 33 | + quiet: boolean; |
| 34 | +} |
| 35 | + |
| 36 | +function validateFilterTestNames(inputDirectory: string, filterTestNames: string[]): TestError[] { |
| 37 | + const availableTests = fs.readdirSync(inputDirectory); |
| 38 | + const missingTests = filterTestNames.filter(name => !availableTests.includes(name)); |
| 39 | + |
| 40 | + if (missingTests.length > 0) { |
| 41 | + return [{ |
| 42 | + testName: missingTests.join(', '), |
| 43 | + type: 'invalid-filter', |
| 44 | + message: `The following test names were not found: ${missingTests.join(', ')}. Available tests: ${availableTests.join(', ')}` |
| 45 | + }]; |
| 46 | + } |
| 47 | + |
| 48 | + return []; |
| 49 | +} |
| 50 | + |
| 51 | +function detectOrphanedOutputs(inputDirectory: string, outputDirectory: string): TestFailure[] { |
| 52 | + if (!fs.existsSync(outputDirectory)) { |
| 53 | + return []; |
| 54 | + } |
| 55 | + |
| 56 | + const inputTests = new Set(fs.readdirSync(inputDirectory)); |
| 57 | + const outputTests = fs.readdirSync(outputDirectory); |
| 58 | + const orphanedOutputs: TestFailure[] = []; |
| 59 | + |
| 60 | + for (const outputTest of outputTests) { |
| 61 | + if (!inputTests.has(outputTest)) { |
| 62 | + orphanedOutputs.push({ |
| 63 | + testName: outputTest, |
| 64 | + type: 'orphaned-output', |
| 65 | + message: `Output folder exists but no corresponding input folder found` |
| 66 | + }); |
| 67 | + } |
| 68 | + } |
| 69 | + |
| 70 | + return orphanedOutputs; |
| 71 | +} |
| 72 | + |
| 73 | +function reportResults(results: ValidationResults): void { |
| 74 | + const totalTests = results.passed.length + results.errors.length + results.failed.length + results.skipped.length; |
| 75 | + console.assert(totalTests > 0, 'No tests found'); |
| 76 | + const errorCount = results.errors.length; |
| 77 | + |
| 78 | + for (const error of results.errors) { |
| 79 | + console.error(`ERROR [${error.testName}]: ${error.message}`); |
| 80 | + } |
| 81 | + |
| 82 | + for (const failure of results.failed) { |
| 83 | + console.error(`FAIL [${failure.testName}]: ${failure.message}`); |
| 84 | + } |
| 85 | + |
| 86 | + let summaryStr = `\n${results.passed.length}/${totalTests} tests passed, ${results.failed.length} failed, ${errorCount} errored`; |
| 87 | + if (results.skipped.length > 0) { |
| 88 | + summaryStr += `, ${results.skipped.length} skipped`; |
| 89 | + } |
| 90 | + console.log(summaryStr); |
| 91 | +} |
| 92 | + |
| 93 | +export class TestRunner { |
| 94 | + constructor(private options: TestRunnerOptions) {} |
| 95 | + |
| 96 | + runTests( |
| 97 | + runSingleTest: (testName: string, inputDir: string, outputDir: string) => ValidationResults |
| 98 | + ): void { |
| 99 | + const inputDirectory = path.resolve(join(this.options.snapshotRoot, 'input')); |
| 100 | + const outputDirectory = path.resolve(join(this.options.snapshotRoot, 'output')); |
| 101 | + const failFast = this.options.failFast ?? false; |
| 102 | + |
| 103 | + const results: ValidationResults = { |
| 104 | + passed: [], |
| 105 | + failed: [], |
| 106 | + errors: [], |
| 107 | + skipped: [] |
| 108 | + }; |
| 109 | + |
| 110 | + // Pre-execution validation: determine test directories to process |
| 111 | + let snapshotDirectories = fs.readdirSync(inputDirectory); |
| 112 | + let isFilterMode = false; |
| 113 | + |
| 114 | + if (this.options.filterTests) { |
| 115 | + // Filter to specific tests |
| 116 | + const filterTestNames = this.options.filterTests.split(',').map(name => name.trim()); |
| 117 | + isFilterMode = true; |
| 118 | + |
| 119 | + // Validate filter test names exist |
| 120 | + const filterErrors = validateFilterTestNames(inputDirectory, filterTestNames); |
| 121 | + if (filterErrors.length > 0) { |
| 122 | + results.errors.push(...filterErrors); |
| 123 | + reportResults(results); |
| 124 | + return; |
| 125 | + } |
| 126 | + |
| 127 | + snapshotDirectories = snapshotDirectories.filter(dir => filterTestNames.includes(dir)); |
| 128 | + } |
| 129 | + |
| 130 | + // In non-filtering mode, detect orphaned outputs that should be cleaned up |
| 131 | + if (!isFilterMode) { |
| 132 | + const orphanedOutputs = detectOrphanedOutputs(inputDirectory, outputDirectory); |
| 133 | + |
| 134 | + // For orphaned outputs in check mode, report as failures |
| 135 | + if (orphanedOutputs.length > 0) { |
| 136 | + results.failed.push(...orphanedOutputs); |
| 137 | + |
| 138 | + if (failFast) { |
| 139 | + reportResults(results); |
| 140 | + return; |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | + |
| 145 | + for (let i = 0; i < snapshotDirectories.length; i++) { |
| 146 | + const testName = snapshotDirectories[i]; |
| 147 | + if (!this.options.quiet) { |
| 148 | + console.log(`Processing test: ${testName}`); |
| 149 | + } |
| 150 | + |
| 151 | + try { |
| 152 | + const testResults = runSingleTest( |
| 153 | + testName, |
| 154 | + inputDirectory, |
| 155 | + outputDirectory, |
| 156 | + ); |
| 157 | + |
| 158 | + // Merge results |
| 159 | + results.passed.push(...testResults.passed); |
| 160 | + results.failed.push(...testResults.failed); |
| 161 | + results.errors.push(...testResults.errors); |
| 162 | + |
| 163 | + // Check for fail-fast condition |
| 164 | + if (failFast && (testResults.failed.length > 0 || testResults.errors.length > 0)) { |
| 165 | + // Track remaining tests as skipped |
| 166 | + for (let j = i + 1; j < snapshotDirectories.length; j++) { |
| 167 | + results.skipped.push(snapshotDirectories[j]); |
| 168 | + } |
| 169 | + reportResults(results); |
| 170 | + return; |
| 171 | + } |
| 172 | + } catch (error) { |
| 173 | + results.errors.push({ |
| 174 | + testName, |
| 175 | + type: 'empty-scip-index', |
| 176 | + message: `Test runner failed: ${error}` |
| 177 | + }); |
| 178 | + |
| 179 | + if (failFast) { |
| 180 | + for (let j = i + 1; j < snapshotDirectories.length; j++) { |
| 181 | + results.skipped.push(snapshotDirectories[j]); |
| 182 | + } |
| 183 | + reportResults(results); |
| 184 | + return; |
| 185 | + } |
| 186 | + } |
| 187 | + } |
| 188 | + |
| 189 | + reportResults(results); |
| 190 | + } |
| 191 | +} |
0 commit comments