diff --git a/package-lock.json b/package-lock.json index 399d917..5438558 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,8 @@ "@google-cloud/vertexai": "^1.1.0", "@google/genai": "^1.50.0", "axios": "^1.13.6", + "crypto": "^1.0.1", + "node-fetch": "^3.3.2", "ts-morph": "^27.0.2" }, "devDependencies": { @@ -177,24 +179,6 @@ "node": ">=14" } }, - "node_modules/@google/genai/node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/@grpc/grpc-js": { "version": "1.14.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", @@ -783,6 +767,13 @@ "node": ">= 8" } }, + "node_modules/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", + "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in.", + "license": "ISC" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -1132,6 +1123,26 @@ "node": ">=14" } }, + "node_modules/gaxios/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/gaxios/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -1344,24 +1355,6 @@ "node": ">=14" } }, - "node_modules/google-gax/node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/google-gax/node_modules/retry-request": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.2.tgz", @@ -1783,23 +1776,21 @@ } }, "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "license": "MIT", "dependencies": { - "whatwg-url": "^5.0.0" + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" }, "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" } }, "node_modules/nodemon": { @@ -2385,6 +2376,26 @@ "node": ">= 6" } }, + "node_modules/teeny-request/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/teeny-request/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", diff --git a/package.json b/package.json index 0e24755..e897c75 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "@google-cloud/vertexai": "^1.1.0", "@google/genai": "^1.50.0", "axios": "^1.13.6", + "crypto": "^1.0.1", + "node-fetch": "^3.3.2", "ts-morph": "^27.0.2" } } diff --git a/vibe-check-runner.js b/vibe-check-runner.js index f56ee94..ec04f83 100644 --- a/vibe-check-runner.js +++ b/vibe-check-runner.js @@ -1,15 +1,14 @@ import { Project, SyntaxKind } from 'ts-morph'; -import fs from 'node:fs'; import fsPromises from 'node:fs/promises'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { GoogleGenAI } from '@google/genai'; +import { pathToFileURL } from 'node:url'; +import crypto from 'node:crypto'; -const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_AI_API_KEY }); +export const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_AI_API_KEY }); -// Constants for scoring - -async function fileOrDirExists(filePath) { +export async function fileOrDirExists(filePath) { try { await fsPromises.stat(filePath); return true; @@ -18,17 +17,15 @@ async function fileOrDirExists(filePath) { } } -const SCORES = { +export const SCORES = { ARCH: 40, TYPE: 30, SECURITY: 20, EFFICIENCY: 10, }; -function getModifiedFiles() { +export function getModifiedFiles() { try { - // In CI (daily run), check files modified in the last 24 hours. - // We filter for non-empty lines that end in .md and are in frontend/ or backend/ const output = execFileSync('sh', ['-c', 'git log --since="24 hours ago" --name-only --pretty=format: | sort | uniq'], { encoding: 'utf-8' }); const allFiles = output.split('\n') .map(f => f.trim()) @@ -43,7 +40,7 @@ function getModifiedFiles() { } } -async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { +export async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { try { const prompt = `Based on the following documentation:\n\n${mdContent}\n\n1. Generate a "Golden Prompt" (a comprehensive instruction for generating a typical module using this technology) in JSON format: {"golden_prompt": "...", "tech": "${tech}"}\n2. Generate a JSON Schema for TS-Morph AST validation rules enforcing DDD/FSD layers and strict typing for this technology. The generated JSON schema must explicitly follow a nested structure compatible with \`analyzeAST\`. Format: {"$schema": "...", "type": "object", "properties": {"forbidden_types": {"contains": {"enum": ["any"]}}}}.\n\nRespond strictly with ONLY a JSON array containing these two objects in order. No markdown wrappers.`; const response = await ai.models.generateContent({ @@ -85,7 +82,7 @@ async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { } } -async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) { +export async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) { try { const prompt = `${goldenPrompt}\n\nConstraints and instructions from the following documentation:\n\n${mdContent}\n\nGenerate ONLY raw code. No markdown formatting, no explanations.`; const response = await ai.models.generateContent({ @@ -110,7 +107,7 @@ async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, } } -function analyzeAST(sourceFile, tech) { +export function analyzeAST(sourceFile, tech) { let score = { arch: SCORES.ARCH, type: SCORES.TYPE, @@ -118,7 +115,6 @@ function analyzeAST(sourceFile, tech) { efficiency: SCORES.EFFICIENCY, }; - // 1. Arch Integrity (40) const decorators = sourceFile.getDescendantsOfKind(SyntaxKind.Decorator); const decoratorNames = decorators.map(d => d.getName()); @@ -127,7 +123,6 @@ function analyzeAST(sourceFile, tech) { score.arch -= 10; } - // DTO Validation check const classDeclarations = sourceFile.getDescendantsOfKind(SyntaxKind.ClassDeclaration); let hasValidation = false; for (const classDecl of classDeclarations) { @@ -138,7 +133,6 @@ function analyzeAST(sourceFile, tech) { } } if (!hasValidation && decoratorNames.length > 0) { - // Note: naive check, only apply if we generated classes score.arch -= 10; } @@ -151,8 +145,6 @@ function analyzeAST(sourceFile, tech) { } } - // FSD/DDD check (Naive representation checking for related imports or folder structure hints in string) - // Check if string contains imports that hint at FSD like '@features', '@entities', '@shared' etc. const imports = sourceFile.getImportDeclarations(); const moduleSpecifiers = imports.map(imp => imp.getModuleSpecifierValue()); const hasFSD = moduleSpecifiers.some(spec => spec.includes('features/') || spec.includes('entities/') || spec.includes('shared/') || spec.includes('domain/')); @@ -161,7 +153,6 @@ function analyzeAST(sourceFile, tech) { score.arch -= 10; } - // 2. Type Safety (30) const parameters = sourceFile.getDescendantsOfKind(SyntaxKind.Parameter); let missingTypes = 0; for (const param of parameters) { @@ -179,24 +170,12 @@ function analyzeAST(sourceFile, tech) { score.type -= 15 * anyKeywords.length; } - // Enforce explicit parameter types - const explicitParameters = sourceFile.getDescendantsOfKind(SyntaxKind.Parameter); - for (const param of explicitParameters) { - if (!param.getTypeNode()) { - score.type -= 5; - } - } - - // Error handling pattern check const tryStatements = sourceFile.getDescendantsOfKind(SyntaxKind.TryStatement); - const catchClauses = sourceFile.getDescendantsOfKind(SyntaxKind.CatchClause); - // If we have functions that do awaiting, they probably should have try/catch const awaitExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.AwaitExpression); if (awaitExpressions.length > 0 && tryStatements.length === 0) { - score.type -= 10; // Penalize lack of error handling + score.type -= 10; } - // 3. Security (20) const stringLiterals = sourceFile.getDescendantsOfKind(SyntaxKind.StringLiteral); for (const literal of stringLiterals) { const text = literal.getText(); @@ -205,10 +184,9 @@ function analyzeAST(sourceFile, tech) { } } - // 4. Efficiency (10) const callExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression); for(const call of callExpressions) { - if(call.getText().includes('readFileSync')) { + if(call.getText().includes('readFileSync') || call.getText().includes('existsSync') || call.getText().includes('writeFileSync')) { score.efficiency -= 10; } } @@ -222,7 +200,7 @@ function analyzeAST(sourceFile, tech) { return { total, breakdown: score }; } -async function runVibeCheck() { +export async function runVibeCheck() { console.log('Running Vibe-Check Runner...'); const modifiedFiles = getModifiedFiles(); @@ -231,9 +209,6 @@ async function runVibeCheck() { return; } - const project = new Project(); - - // Configure git user for commits try { execFileSync('git', ['config', '--global', 'user.name', 'github-actions[bot]']); execFileSync('git', ['config', '--global', 'user.email', 'github-actions[bot]@users.noreply.github.com']); @@ -241,10 +216,9 @@ async function runVibeCheck() { console.warn('Failed to configure git user. If running locally, this is expected.'); } + const filesByTech = {}; for (const file of modifiedFiles) { - console.log(`Processing ${file}...`); - - if (!fs.existsSync(file)) { + if (!await fileOrDirExists(file)) { console.log(`File ${file} does not exist. Skipping.`); continue; } @@ -256,7 +230,6 @@ async function runVibeCheck() { else if (file.includes('/express/')) tech = 'express'; else if (file.includes('/nodejs/')) tech = 'nodejs'; else { - // Fallback const parts = file.split('/'); if (parts.length > 1) { tech = parts[1]; @@ -265,34 +238,60 @@ async function runVibeCheck() { } } - const mdContent = await fsPromises.readFile(file, 'utf-8'); + if (!filesByTech[tech]) filesByTech[tech] = []; + const content = await fsPromises.readFile(file, 'utf-8'); + filesByTech[tech].push({ file, content, tech }); + } - await syncBenchmarks(tech, mdContent); + for (const tech in filesByTech) { + const combinedContent = filesByTech[tech].map(item => item.content).join('\n\n--- \n\n'); + await syncBenchmarks(tech, combinedContent); + } - const suitePath = path.join('benchmarks', 'suites', `${tech}.json`); - if (!fs.existsSync(suitePath)) { - console.log(`No benchmark suite found for ${tech}. Skipping.`); - continue; + const filePromises = []; + for (const tech in filesByTech) { + for (const item of filesByTech[tech]) { + filePromises.push((async () => { + const { file, content, tech } = item; + const suitePath = path.join('benchmarks', 'suites', `${tech}.json`); + + if (!await fileOrDirExists(suitePath)) { + return { file, error: `No benchmark suite found for ${tech}. Skipping.` }; + } + + const suiteConfig = JSON.parse(await fsPromises.readFile(suitePath, 'utf-8')); + const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, content); + + if (!generatedCode) { + return { file, error: `Failed to generate code for ${tech}.` }; + } + + const localProject = new Project(); + const tempFileName = `temp_${crypto.randomUUID()}.ts`; + const sourceFile = localProject.createSourceFile(tempFileName, generatedCode, { overwrite: true }); + + const { total: score, breakdown } = analyzeAST(sourceFile, tech); + return { file, score, breakdown, generatedCode }; + })()); } + } - const suiteConfig = JSON.parse(await fsPromises.readFile(suitePath, 'utf-8')); + const results = await Promise.all(filePromises); - const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, mdContent); + let hasFailures = false; - if (!generatedCode) { - console.error(`Failed to generate code for ${tech}.`); + for (const result of results) { + if (result.error) { + console.log(result.error); continue; } - const sourceFile = project.createSourceFile(`temp_${tech}.ts`, generatedCode, { overwrite: true }); - const { total: score, breakdown } = analyzeAST(sourceFile, tech); - + const { file, score, breakdown, generatedCode } = result; console.log(`Fidelity Score for ${file}: ${score}%`); console.log(`Breakdown:`, breakdown); if (score >= 95) { console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`); - let content = await fsPromises.readFile(file, 'utf-8'); if (!content.includes('[![Vibe-Coding Verified]')) { content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# '); @@ -301,11 +300,15 @@ async function runVibeCheck() { try { execFileSync('git', ['add', file]); - try { execFileSync('sh', ['-c', 'git add benchmarks/suites/*.json benchmarks/criteria/*.json 2>/dev/null || true']); } catch (e) {} - // Only commit if there are changes (badge might already be there) + try { + const suitesPath = path.join('benchmarks', 'suites', '*.json'); + const criteriaPath = path.join('benchmarks', 'criteria', '*.json'); + execFileSync('sh', ['-c', `git add ${suitesPath} ${criteriaPath} 2>/dev/null || true`]); + } catch (e) {} + const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }); if (status.includes(file) || status.includes('benchmarks/')) { - execFileSync('git', ['commit', '-m', '[chore: benchmark-sync]']); + execFileSync('git', ['commit', '-m', '[chore: fidelity-pass]']); execFileSync('git', ['push', 'origin', 'HEAD:main']); } else { console.log(`Badge already present in ${file}, skipping commit.`); @@ -313,9 +316,9 @@ async function runVibeCheck() { } catch (err) { console.error('Failed to commit or push:', err.message); } - } else { console.error(`❌ Validation failed for ${file}. Score below 95%.`); + hasFailures = true; const reportDir = path.join('benchmarks', 'logs'); if (!await fileOrDirExists(reportDir)) await fsPromises.mkdir(reportDir, { recursive: true }); @@ -332,10 +335,14 @@ async function runVibeCheck() { } catch (err) { console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message); } - - process.exitCode = 1; } } + + if (hasFailures) { + process.exitCode = 1; + } } -runVibeCheck().catch(console.error); \ No newline at end of file +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + runVibeCheck().catch(console.error); +}