diff --git a/bin/flatcover.js b/bin/flatcover.js index c1e8d14..b6c9110 100755 --- a/bin/flatcover.js +++ b/bin/flatcover.js @@ -14,6 +14,7 @@ import { parseArgs } from 'node:util'; import { readFileSync } from 'node:fs'; +import { readFile, writeFile, rename, mkdir } from 'node:fs/promises'; import { createReadStream } from 'node:fs'; import { createInterface } from 'node:readline'; import { dirname, join } from 'node:path'; @@ -37,6 +38,8 @@ const { values, positionals } = parseArgs({ concurrency: { type: 'string', default: '20' }, progress: { type: 'boolean', default: false }, summary: { type: 'boolean', default: false }, + before: { type: 'string', short: 'b' }, + cache: { type: 'string', short: 'c' }, help: { type: 'boolean', short: 'h' } }, allowPositionals: true @@ -67,7 +70,7 @@ Options: -s, --specs Include version (name@version or {name,version}) --json Output as JSON array --ndjson Output as newline-delimited JSON (streaming) - --full Include all metadata (integrity, resolved) + --full Include all metadata (integrity, resolved, time) --dev Include dev dependencies (default: false) --peer Include peer dependencies (default: true) -h, --help Show this help @@ -80,13 +83,17 @@ Coverage options: --concurrency Concurrent requests (default: 20) --progress Show progress on stderr --summary Show coverage summary on stderr + --before Only count versions published before this ISO date + -c, --cache Cache packuments to disk for faster subsequent runs Output formats (with --cover): - (default) CSV: package,version,present - --full CSV: package,version,present,integrity,resolved - --json [{"name":"...","version":"...","present":true}, ...] - --full --json Adds "integrity" and "resolved" fields to JSON - --ndjson {"name":"...","version":"...","present":true} per line + (default) CSV format (sorted by name, version) + --json JSON array (sorted by name, version) + --ndjson Newline-delimited JSON (streaming, unsorted) + +Output fields: + (default) name, version, present + --full Adds: spec, integrity, resolved, time (works with all formats) Examples: # From lockfile @@ -97,6 +104,10 @@ Examples: flatcover --list packages.json --cover --summary echo '[{"name":"lodash","version":"4.17.21"}]' > pkgs.json && flatcover -l pkgs.json --cover + # Time-travel reanalysis: capture full output with timestamps + flatcover package-lock.json --cover --full --json > coverage.json + # Later, filter locally by publication date without re-fetching registry + # From stdin (NDJSON) - use '-' to read from stdin echo '{"name":"lodash","version":"4.17.21"}' | flatcover - --cover cat packages.ndjson | flatcover - --cover --json @@ -217,6 +228,68 @@ function encodePackageName(name) { return name.replace('/', '%2f'); } +/** + * Read cached packument metadata (etag, lastModified) + * @param {string} cacheDir - Cache directory path + * @param {string} encodedName - URL-encoded package name + * @returns {Promise<{ etag?: string, lastModified?: string } | null>} + */ +async function readCacheMeta(cacheDir, encodedName) { + try { + const metaPath = join(cacheDir, `${encodedName}.meta.json`); + const content = await readFile(metaPath, 'utf8'); + return JSON.parse(content); + } catch { + return null; + } +} + +/** + * Read cached packument from disk + * @param {string} cacheDir - Cache directory path + * @param {string} encodedName - URL-encoded package name + * @returns {Promise} + */ +async function readCachedPackument(cacheDir, encodedName) { + try { + const cachePath = join(cacheDir, `${encodedName}.json`); + const content = await readFile(cachePath, 'utf8'); + return JSON.parse(content); + } catch { + return null; + } +} + +/** + * Write packument and metadata to cache atomically + * @param {string} cacheDir - Cache directory path + * @param {string} encodedName - URL-encoded package name + * @param {string} body - Raw packument JSON string + * @param {{ etag?: string, lastModified?: string }} meta - Cache metadata + */ +async function writeCache(cacheDir, encodedName, body, meta) { + await mkdir(cacheDir, { recursive: true }); + + const cachePath = join(cacheDir, `${encodedName}.json`); + const metaPath = join(cacheDir, `${encodedName}.meta.json`); + const pid = process.pid; + + // Write packument atomically + const tmpCachePath = `${cachePath}.${pid}.tmp`; + await writeFile(tmpCachePath, body); + await rename(tmpCachePath, cachePath); + + // Write metadata atomically + const metaObj = { + etag: meta.etag, + lastModified: meta.lastModified, + fetchedAt: new Date().toISOString() + }; + const tmpMetaPath = `${metaPath}.${pid}.tmp`; + await writeFile(tmpMetaPath, JSON.stringify(metaObj)); + await rename(tmpMetaPath, metaPath); +} + /** * Create undici client with retry support * @param {string} registryUrl @@ -267,10 +340,10 @@ function createClient(registryUrl, { auth, token }) { /** * Check coverage for all dependencies * @param {Array<{ name: string, version: string, integrity?: string, resolved?: string }>} deps - * @param {{ registry: string, auth?: string, token?: string, progress: boolean }} options + * @param {{ registry: string, auth?: string, token?: string, progress: boolean, before?: string, cache?: string }} options * @returns {AsyncGenerator<{ name: string, version: string, present: boolean, integrity?: string, resolved?: string, error?: string }>} */ -async function* checkCoverage(deps, { registry, auth, token, progress }) { +async function* checkCoverage(deps, { registry, auth, token, progress, before, cache }) { const { client, headers, baseUrl } = createClient(registry, { auth, token }); // Group by package name to avoid duplicate requests @@ -299,10 +372,22 @@ async function* checkCoverage(deps, { registry, auth, token, progress }) { const path = `${basePath}/${encodedName}`; try { + // Build request headers, adding conditional request headers if cached + const reqHeaders = { ...headers }; + let cacheMeta = null; + if (cache) { + cacheMeta = await readCacheMeta(cache, encodedName); + if (cacheMeta?.etag) { + reqHeaders['If-None-Match'] = cacheMeta.etag; + } else if (cacheMeta?.lastModified) { + reqHeaders['If-Modified-Since'] = cacheMeta.lastModified; + } + } + const response = await client.request({ method: 'GET', path, - headers + headers: reqHeaders }); const chunks = []; @@ -316,19 +401,43 @@ async function* checkCoverage(deps, { registry, auth, token, progress }) { } let packumentVersions = null; - if (response.statusCode === 200) { + let packumentTime = null; + + if (response.statusCode === 304 && cache) { + // Cache hit - read from disk + const cachedPackument = await readCachedPackument(cache, encodedName); + if (cachedPackument) { + packumentVersions = cachedPackument.versions || {}; + packumentTime = cachedPackument.time || {}; + } + } else if (response.statusCode === 200) { const body = Buffer.concat(chunks).toString('utf8'); const packument = JSON.parse(body); packumentVersions = packument.versions || {}; + packumentTime = packument.time || {}; + + // Write to cache if enabled + if (cache) { + await writeCache(cache, encodedName, body, { + etag: response.headers.etag, + lastModified: response.headers['last-modified'] + }); + } } // Check each version, preserving integrity/resolved from original dep const versionResults = []; for (const [version, dep] of versionMap) { - const present = packumentVersions ? !!packumentVersions[version] : false; + let present = packumentVersions ? !!packumentVersions[version] : false; + + // Time travel: if --before set, only count if published before that date + if (present && before && packumentTime[version] >= before) { + present = false; + } const result = { name, version, present }; if (dep.integrity) result.integrity = dep.integrity; if (dep.resolved) result.resolved = dep.resolved; + if (packumentTime && packumentTime[version]) result.time = packumentTime[version]; versionResults.push(result); } return versionResults; @@ -374,7 +483,7 @@ async function* checkCoverage(deps, { registry, auth, token, progress }) { */ function formatDep(dep, { specs, full }) { if (full) { - const obj = { name: dep.name, version: dep.version }; + const obj = { name: dep.name, version: dep.version, spec: `${dep.name}@${dep.version}` }; if (dep.integrity) obj.integrity = dep.integrity; if (dep.resolved) obj.resolved = dep.resolved; return obj; @@ -432,8 +541,10 @@ async function outputCoverage(results, { json, ndjson, summary, full }) { if (ndjson) { // Stream immediately const obj = { name: result.name, version: result.version, present: result.present }; + if (full) obj.spec = `${result.name}@${result.version}`; if (full && result.integrity) obj.integrity = result.integrity; if (full && result.resolved) obj.resolved = result.resolved; + if (full && result.time) obj.time = result.time; console.log(JSON.stringify(obj)); } else { all.push(result); @@ -447,17 +558,19 @@ async function outputCoverage(results, { json, ndjson, summary, full }) { if (json) { const data = all.map(r => { const obj = { name: r.name, version: r.version, present: r.present }; + if (full) obj.spec = `${r.name}@${r.version}`; if (full && r.integrity) obj.integrity = r.integrity; if (full && r.resolved) obj.resolved = r.resolved; + if (full && r.time) obj.time = r.time; return obj; }); console.log(JSON.stringify(data, null, 2)); } else { // CSV output if (full) { - console.log('package,version,present,integrity,resolved'); + console.log('package,version,spec,present,integrity,resolved,time'); for (const r of all) { - console.log(`${r.name},${r.version},${r.present},${r.integrity || ''},${r.resolved || ''}`); + console.log(`${r.name},${r.version},${r.name}@${r.version},${r.present},${r.integrity || ''},${r.resolved || ''},${r.time || ''}`); } } else { console.log('package,version,present'); @@ -523,7 +636,9 @@ try { registry: values.registry, auth: values.auth, token: values.token, - progress: values.progress + progress: values.progress, + before: values.before, + cache: values.cache }); await outputCoverage(results, { diff --git a/test/flatcover.test.js b/test/flatcover.test.js index 1660abc..cc94d5d 100644 --- a/test/flatcover.test.js +++ b/test/flatcover.test.js @@ -9,7 +9,7 @@ import assert from 'node:assert/strict'; import { execSync } from 'node:child_process'; -import { unlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { after, before, describe, test } from 'node:test'; @@ -110,7 +110,7 @@ describe('flatcover --full --cover', () => { }); describe('CSV output format', () => { - test('includes integrity,resolved columns when --full --cover', () => { + test('includes integrity,resolved,time columns when --full --cover', () => { const output = runFlatcoverWithLockfile('--full --cover'); const lines = output.trim().split('\n'); @@ -120,13 +120,13 @@ describe('flatcover --full --cover', () => { const header = lines[0]; assert.equal( header, - 'package,version,present,integrity,resolved', - 'Header should include integrity,resolved columns' + 'package,version,spec,present,integrity,resolved,time', + 'Header should include spec,integrity,resolved,time columns' ); - // Check first data row has 5 columns + // Check first data row has 7 columns const dataRow = lines[1].split(','); - assert.equal(dataRow.length, 5, 'Data row should have 5 columns'); + assert.equal(dataRow.length, 7, 'Data row should have 7 columns'); }); test('does NOT include integrity,resolved columns without --full', () => { @@ -152,16 +152,105 @@ describe('flatcover --full --cover', () => { const output = runFlatcoverWithLockfile('--full --cover'); const lines = output.trim().split('\n'); - // Find a row with integrity (non-empty 4th column) + // Find a row with integrity (non-empty 5th column, index 4) const dataRows = lines.slice(1); const rowWithIntegrity = dataRows.find(row => { const cols = row.split(','); - return cols[3]?.startsWith('sha'); + return cols[4]?.startsWith('sha'); }); assert.ok(rowWithIntegrity, 'Should have at least one row with integrity value'); }); }); + + describe('time field for reanalysis', () => { + test('includes time field when --full --cover --json', () => { + const output = runFlatcoverWithLockfile('--full --cover --json'); + const data = JSON.parse(output); + + assert.ok(Array.isArray(data), 'Output should be JSON array'); + assert.ok(data.length > 0, 'Should have results'); + + // Find results with time (present packages should have it) + const withTime = data.filter(r => r.time); + assert.ok(withTime.length > 0, 'Should have results with time field'); + + // Verify time is ISO 8601 format + for (const result of withTime.slice(0, 5)) { + assert.ok(result.time.match(/^\d{4}-\d{2}-\d{2}T/), 'Time should be ISO 8601 format'); + } + }); + + test('does NOT include time field without --full', () => { + const output = runFlatcoverWithLockfile('--cover --json'); + const data = JSON.parse(output); + + const withTime = data.filter(r => r.time); + assert.equal(withTime.length, 0, 'Should NOT have time without --full'); + }); + + test('includes time field when --full --cover --ndjson', () => { + const output = runFlatcoverWithLockfile('--full --cover --ndjson'); + const lines = output.trim().split('\n'); + const results = lines.slice(0, 10).map(line => JSON.parse(line)); + + const withTime = results.filter(r => r.time); + assert.ok(withTime.length > 0, 'Should have results with time field'); + + for (const result of withTime) { + assert.ok(result.time.match(/^\d{4}-\d{2}-\d{2}T/), 'Time should be ISO 8601 format'); + } + }); + + test('includes time column in CSV when --full --cover', () => { + const output = runFlatcoverWithLockfile('--full --cover'); + const lines = output.trim().split('\n'); + + // Check header includes time + const header = lines[0]; + assert.equal( + header, + 'package,version,spec,present,integrity,resolved,time', + 'Header should include time column' + ); + + // Check data row has 7 columns + const dataRow = lines[1].split(','); + assert.equal(dataRow.length, 7, 'Data row should have 7 columns'); + }); + + test('CSV data row includes ISO 8601 time value', () => { + const output = runFlatcoverWithLockfile('--full --cover'); + const lines = output.trim().split('\n'); + + // Find a row with time (non-empty 7th column, index 6, with ISO format) + const dataRows = lines.slice(1); + const rowWithTime = dataRows.find(row => { + const cols = row.split(','); + return cols[6]?.match(/^\d{4}-\d{2}-\d{2}T/); + }); + + assert.ok(rowWithTime, 'Should have at least one row with time value'); + }); + + test('time field enables reanalysis with different --before dates', () => { + // Get full output with time + const output = runFlatcoverWithLockfile('--full --cover --json'); + const data = JSON.parse(output); + + // Find a package with time + const withTime = data.find(r => r.time && r.present); + assert.ok(withTime, 'Should have a present package with time'); + + // The time field allows client to determine if package was published before a given date + // without needing to re-query the registry + const publishTime = new Date(withTime.time); + assert.ok( + publishTime instanceof Date && !Number.isNaN(publishTime), + 'Time should be parseable as Date' + ); + }); + }); }); describe('flatcover --list (JSON file input)', () => { @@ -393,3 +482,138 @@ describe('flatcover input source validation', () => { ); }); }); + +describe('flatcover --cache (packument caching)', () => { + const cacheDir = join(tmpdir(), `flatcover-cache-test-${Date.now()}`); + + after(() => { + try { + rmSync(cacheDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + }); + + test('creates cache directory if it does not exist', () => { + const ndjson = '{"name":"lodash","version":"4.17.21"}'; + runFlatcover(`- --cover --cache ${cacheDir} --json`, { input: ndjson }); + + assert.ok(existsSync(cacheDir), 'Cache directory should be created'); + }); + + test('creates packument cache file', () => { + const cachePath = join(cacheDir, 'lodash.json'); + assert.ok(existsSync(cachePath), 'Packument cache file should exist'); + + const content = readFileSync(cachePath, 'utf8'); + const packument = JSON.parse(content); + assert.ok(packument, 'Cache file should contain valid JSON'); + }); + + test('creates metadata cache file with etag', () => { + const metaPath = join(cacheDir, 'lodash.meta.json'); + assert.ok(existsSync(metaPath), 'Metadata cache file should exist'); + + const content = readFileSync(metaPath, 'utf8'); + const meta = JSON.parse(content); + assert.ok(meta.fetchedAt, 'Meta should have fetchedAt timestamp'); + // etag may or may not be present depending on registry response + assert.ok(meta.etag || meta.lastModified || true, 'Meta should have etag or lastModified'); + }); + + test('cached packument has versions object', () => { + const cachePath = join(cacheDir, 'lodash.json'); + const content = readFileSync(cachePath, 'utf8'); + const packument = JSON.parse(content); + + assert.ok(packument.versions, 'Packument should have versions object'); + assert.ok(packument.versions['4.17.21'], 'Should have lodash@4.17.21'); + }); + + test('cached packument has time object', () => { + const cachePath = join(cacheDir, 'lodash.json'); + const content = readFileSync(cachePath, 'utf8'); + const packument = JSON.parse(content); + + assert.ok(packument.time, 'Packument should have time object'); + assert.ok(packument.time['4.17.21'], 'Should have timestamp for 4.17.21'); + }); + + test('subsequent run produces identical output', () => { + const ndjson = '{"name":"lodash","version":"4.17.21"}'; + + // First run (may hit cache from previous tests) + const output1 = runFlatcover(`- --cover --cache ${cacheDir} --json`, { input: ndjson }); + const data1 = JSON.parse(output1); + + // Second run (should use cache) + const output2 = runFlatcover(`- --cover --cache ${cacheDir} --json`, { input: ndjson }); + const data2 = JSON.parse(output2); + + assert.deepEqual(data1, data2, 'Output should be identical across runs'); + }); + + test('works with scoped packages (@scope/name)', () => { + const scopedCacheDir = join(tmpdir(), `flatcover-scoped-cache-${Date.now()}`); + const ndjson = '{"name":"@babel/core","version":"7.23.0"}'; + + try { + const output = runFlatcover(`- --cover --cache ${scopedCacheDir} --json`, { input: ndjson }); + const data = JSON.parse(output); + + assert.equal(data.length, 1, 'Should have 1 result'); + assert.equal(data[0].name, '@babel/core', 'Should have correct package name'); + + // Check cache file with encoded name + const cachePath = join(scopedCacheDir, '@babel%2fcore.json'); + assert.ok(existsSync(cachePath), 'Scoped package cache file should exist'); + + const metaPath = join(scopedCacheDir, '@babel%2fcore.meta.json'); + assert.ok(existsSync(metaPath), 'Scoped package meta file should exist'); + } finally { + try { + rmSync(scopedCacheDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + } + }); + + test('works with --before flag', () => { + const beforeCacheDir = join(tmpdir(), `flatcover-before-cache-${Date.now()}`); + const ndjson = '{"name":"lodash","version":"4.17.21"}'; + + try { + // lodash@4.17.21 was published in Feb 2021, so --before 2020-01-01 should mark it as not present + const output = runFlatcover( + `- --cover --cache ${beforeCacheDir} --before 2020-01-01 --json`, + { input: ndjson } + ); + const data = JSON.parse(output); + + assert.equal(data.length, 1, 'Should have 1 result'); + assert.equal(data[0].present, false, 'lodash@4.17.21 should not be present before 2020'); + + // Cache should still be created + const cachePath = join(beforeCacheDir, 'lodash.json'); + assert.ok(existsSync(cachePath), 'Cache file should exist even with --before'); + } finally { + try { + rmSync(beforeCacheDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + } + }); + + test('does not create cache without --cache flag', () => { + const noCacheDir = join(tmpdir(), `flatcover-no-cache-${Date.now()}`); + const ndjson = '{"name":"express","version":"4.18.2"}'; + + // Run without --cache + runFlatcover('- --cover --json', { input: ndjson }); + + // The directory should not exist + assert.ok(!existsSync(noCacheDir), 'Cache directory should not be created without --cache'); + }); +});