diff --git a/docs/ci-integration.md b/docs/ci-integration.md index 9a18e59..95443ad 100644 --- a/docs/ci-integration.md +++ b/docs/ci-integration.md @@ -79,6 +79,12 @@ url: https://docs.example.com # options: # maxLinksToTest: 50 # samplingStrategy: deterministic + +# Optional: test specific pages (implies samplingStrategy: curated) +# pages: +# - https://docs.example.com/quickstart +# - url: https://docs.example.com/api/auth +# tag: api-reference ``` ### Config resolution diff --git a/docs/quick-start.md b/docs/quick-start.md index 8edb85b..7891c9c 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -95,6 +95,36 @@ You can combine this with `--checks` to run a single check against a single page npx afdocs check https://docs.example.com/api/auth --sampling none --checks rendering-strategy ``` +## Check a specific set of pages + +If you want to check a handful of pages without running full discovery, pass them directly with `--urls`: + +```bash +npx afdocs check https://docs.example.com --urls https://docs.example.com/quickstart,https://docs.example.com/api/auth +``` + +This skips page discovery and runs all checks against exactly those URLs. You can tag pages for grouped scoring by defining them in a config file: + +```yaml +# agent-docs.config.yml +url: https://docs.example.com +pages: + - url: https://docs.example.com/quickstart + tag: getting-started + - url: https://docs.example.com/tutorials/first-app + tag: getting-started + - url: https://docs.example.com/api/auth + tag: api-reference + - url: https://docs.example.com/api/webhooks + tag: api-reference +``` + +```bash +npx afdocs check --format scorecard +``` + +The scorecard will include a Tag Scores section showing how each group of pages scores, with a per-check breakdown of what's passing and failing within each tag. The JSON output (`--format json --score`) includes full per-page detail for each tag. See [Config File Reference](/reference/config-file) for the full `pages` schema. + ## Get consistent results between runs By default, AFDocs randomly samples pages, so results can vary between runs. For reproducible results (useful when verifying a fix), use deterministic sampling: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 0120dcc..3ac59f8 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -77,15 +77,17 @@ Some checks depend on others. If you include a check without its dependency, the ### Sampling -| Flag | Default | Description | -| ----------------------- | -------- | ----------------------------------------------------------- | -| `--sampling ` | `random` | URL sampling strategy: `random`, `deterministic`, or `none` | -| `--max-links ` | `50` | Maximum number of pages to sample | +| Flag | Default | Description | +| ----------------------- | -------- | ---------------------------------------------------------------------------- | +| `--sampling ` | `random` | URL sampling strategy: `random`, `deterministic`, `curated`, or `none` | +| `--max-links ` | `50` | Maximum number of pages to sample | +| `--urls ` | | Comma-separated page URLs for curated scoring (implies `--sampling curated`) | **Sampling strategies:** - **`random`**: Shuffle discovered URLs and take the first N. Fast and broad, but results vary between runs. Useful for spot-checking pages across a large corpus. - **`deterministic`**: Sort discovered URLs alphabetically and pick an even spread. Produces the same sample on repeated runs as long as the URL set is stable. Useful for CI or when verifying a fix. +- **`curated`**: Test a specific set of pages listed in the config file's `pages` field or passed via `--urls`. Skips discovery entirely. Useful for ongoing monitoring of representative pages or focused evaluation of specific sections. - **`none`**: Skip discovery entirely. Only check the URL you pass on the command line. ```bash @@ -95,6 +97,9 @@ afdocs check https://docs.example.com --sampling deterministic # Check a single page afdocs check https://docs.example.com/api/auth --sampling none +# Test specific pages without a config file +afdocs check https://docs.example.com --urls https://docs.example.com/quickstart,https://docs.example.com/api/auth + # Sample fewer pages for a quicker run afdocs check https://docs.example.com --max-links 10 diff --git a/docs/reference/config-file.md b/docs/reference/config-file.md index 8489d5c..e3f269e 100644 --- a/docs/reference/config-file.md +++ b/docs/reference/config-file.md @@ -28,6 +28,12 @@ options: thresholds: pass: 50000 fail: 100000 + +# Optional: test specific pages instead of discovering via llms.txt/sitemap +# pages: +# - https://docs.example.com/quickstart +# - url: https://docs.example.com/api/auth +# tag: api-reference ``` ## Fields @@ -49,13 +55,40 @@ Override default runner options. All fields are optional: | Field | Default | Description | | ------------------ | -------- | ---------------------------------------------------- | | `maxLinksToTest` | `50` | Maximum number of pages to sample | -| `samplingStrategy` | `random` | `random`, `deterministic`, or `none` | +| `samplingStrategy` | `random` | `random`, `deterministic`, `curated`, or `none` | | `maxConcurrency` | `3` | Maximum concurrent HTTP requests | | `requestDelay` | `200` | Delay between requests in milliseconds | | `requestTimeout` | `30000` | Timeout for individual HTTP requests in milliseconds | | `thresholds.pass` | `50000` | Page size pass threshold in characters | | `thresholds.fail` | `100000` | Page size fail threshold in characters | +### `pages` (optional) + +A list of specific page URLs to test. When `pages` is present and no `samplingStrategy` is explicitly set, the strategy defaults to `curated`, which skips discovery and tests exactly the listed pages. + +Each entry can be a plain URL string or an object with `url` and an optional `tag` for grouped scoring: + +```yaml +url: https://docs.example.com + +pages: + # Plain URL strings + - https://docs.example.com/quickstart + - https://docs.example.com/install + + # Objects with tags for grouped scoring + - url: https://docs.example.com/api/auth + tag: api-reference + - url: https://docs.example.com/api/users + tag: api-reference +``` + +When pages have tags, the scorecard and JSON output include per-tag aggregate scores, making it easy to compare agent-friendliness across sections of your documentation. + +Tags are optional and can be mixed with plain URL strings. Pages without tags are included in the overall score but don't appear in any tag group. + +Note that `maxLinksToTest` does not apply to curated pages; all listed pages are tested. + ## Config resolution The config loader searches for `agent-docs.config.yml` (or `.yaml`) starting from the current working directory and walking up the directory tree, similar to how ESLint and Prettier find their config files. This means the config works whether you're running the CLI from your project root or running a test file from a subdirectory. diff --git a/docs/reference/programmatic-api.md b/docs/reference/programmatic-api.md index 4873bc3..87acede 100644 --- a/docs/reference/programmatic-api.md +++ b/docs/reference/programmatic-api.md @@ -42,6 +42,15 @@ const report = await runChecks('https://docs.example.com', { fail: 100000, }, }); + +// Or test specific pages with curated sampling: +const curatedReport = await runChecks('https://docs.example.com', { + samplingStrategy: 'curated', + curatedPages: [ + 'https://docs.example.com/quickstart', + { url: 'https://docs.example.com/api/auth', tag: 'api-reference' }, + ], +}); ``` All options are optional. The defaults match the CLI defaults. @@ -90,6 +99,8 @@ import type { RunnerOptions, CheckOptions, AgentDocsConfig, + CuratedPageEntry, + PageConfigEntry, } from 'afdocs'; ``` diff --git a/docs/reference/scoring-api.md b/docs/reference/scoring-api.md index f97ff5c..2d5e02f 100644 --- a/docs/reference/scoring-api.md +++ b/docs/reference/scoring-api.md @@ -42,6 +42,43 @@ This is the same function; the subpath is provided for consumers who want a narr | `diagnostics` | `Diagnostic[]` | Interaction diagnostics that fired | | `caps` | `ScoreCap[]` | Score caps that were applied | | `resolutions` | `Record` | Fix suggestions keyed by check ID | +| `tagScores` | `Record` | Per-tag aggregate scores (present when curated pages have tags) | + +## TagScore + +When curated pages have tags, each `TagScore` contains the aggregate score plus a per-check breakdown showing exactly which checks contributed and how each page fared: + +| Field | Type | Description | +| ----------- | --------------------- | -------------------------------------------------------------- | +| `score` | `number` | Aggregate score for this tag (0-100) | +| `grade` | `Grade` | Letter grade | +| `pageCount` | `number` | Number of pages tagged with this tag | +| `checks` | `TagCheckBreakdown[]` | Per-check breakdown with weight, proportion, and page statuses | + +Each `TagCheckBreakdown` contains: + +| Field | Type | Description | +| ------------ | ---------------------------------------- | -------------------------------------------------- | +| `checkId` | `string` | The check ID | +| `category` | `string` | The check's category | +| `weight` | `number` | The check's effective weight in the scoring system | +| `proportion` | `number` | 0-1 proportion earned for this tag's pages | +| `pages` | `Array<{ url: string; status: string }>` | Per-page status within this check | + +```ts +const score = computeScore(report); +if (score.tagScores) { + for (const [tag, tagScore] of Object.entries(score.tagScores)) { + console.log(`${tag}: ${tagScore.score}/100 (${tagScore.grade})`); + for (const check of tagScore.checks) { + if (check.proportion < 1) { + const failing = check.pages.filter((p) => p.status === 'fail'); + console.log(` ${check.checkId}: ${failing.length} failing pages`); + } + } + } +} +``` ## Grade conversion @@ -62,6 +99,8 @@ import type { ScoreResult, CheckScore, CategoryScore, + TagScore, + TagCheckBreakdown, ScoreCap, Diagnostic, DiagnosticSeverity, // 'info' | 'warning' | 'critical' diff --git a/examples/agent-docs.config.yml b/examples/agent-docs.config.yml index 1b6f83e..aeeea47 100644 --- a/examples/agent-docs.config.yml +++ b/examples/agent-docs.config.yml @@ -11,3 +11,13 @@ url: https://docs.example.com # options: # maxLinksToTest: 50 # samplingStrategy: deterministic + +# Optional: test specific pages instead of discovering via llms.txt/sitemap. +# When pages is present and no samplingStrategy is set, defaults to 'curated'. +# pages: +# - https://docs.example.com/quickstart +# - https://docs.example.com/install +# - url: https://docs.example.com/api/auth +# tag: api-reference +# - url: https://docs.example.com/api/users +# tag: api-reference diff --git a/src/cli/commands/check.ts b/src/cli/commands/check.ts index 199c4f1..c600d1e 100644 --- a/src/cli/commands/check.ts +++ b/src/cli/commands/check.ts @@ -3,13 +3,13 @@ import { normalizeUrl, runChecks } from '../../runner.js'; import { formatText } from '../formatters/text.js'; import { formatJson } from '../formatters/json.js'; import { formatScorecard } from '../formatters/scorecard.js'; -import type { SamplingStrategy } from '../../types.js'; -import { findConfig } from '../../helpers/config.js'; +import type { PageConfigEntry, SamplingStrategy } from '../../types.js'; +import { findConfig, validatePages } from '../../helpers/config.js'; // Ensure all checks are registered import '../../checks/index.js'; -const SAMPLING_STRATEGIES = ['random', 'deterministic', 'none'] as const; +const SAMPLING_STRATEGIES = ['random', 'deterministic', 'curated', 'none'] as const; const FORMAT_OPTIONS = ['text', 'json', 'scorecard'] as const; export function registerCheckCommand(program: Command): void { @@ -22,7 +22,14 @@ export function registerCheckCommand(program: Command): void { .option('--max-concurrency ', 'Maximum concurrent requests') .option('--request-delay ', 'Delay between requests in ms') .option('--max-links ', 'Maximum links to test') - .option('--sampling ', 'URL sampling strategy: random, deterministic, or none') + .option( + '--sampling ', + 'URL sampling strategy: random, deterministic, curated, or none', + ) + .option( + '--urls ', + 'Comma-separated page URLs for curated scoring (implies --sampling curated)', + ) .option('--pass-threshold ', 'Pass threshold in characters') .option('--fail-threshold ', 'Fail threshold in characters') .option('-v, --verbose', 'Show per-page details for checks with issues') @@ -39,8 +46,49 @@ export function registerCheckCommand(program: Command): void { return; } - // Resolve URL: CLI arg > config url > error - const resolvedUrl = rawUrl ?? config?.url; + // Determine curated pages and sampling strategy (before URL resolution, + // since curated pages can provide a fallback base URL) + let curatedPages: PageConfigEntry[] | undefined; + let samplingRaw: string; + + if (opts.urls) { + // --urls flag: parse comma-separated URLs, force curated strategy + const rawUrls = (opts.urls as string) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + if (rawUrls.length === 0) { + process.stderr.write('Error: --urls requires at least one URL.\n'); + process.exitCode = 1; + return; + } + try { + validatePages(rawUrls, '--urls'); + } catch (err) { + process.stderr.write(`Error: ${(err as Error).message}\n`); + process.exitCode = 1; + return; + } + curatedPages = rawUrls; + samplingRaw = 'curated'; + } else if (config?.pages && config.pages.length > 0) { + // Config has pages: use them, default to curated unless explicitly overridden + curatedPages = config.pages; + samplingRaw = + (opts.sampling as string | undefined) ?? config?.options?.samplingStrategy ?? 'curated'; + } else { + // No curated pages: standard behavior + samplingRaw = + (opts.sampling as string | undefined) ?? config?.options?.samplingStrategy ?? 'random'; + } + + // Resolve URL: CLI arg > config url > first curated page origin > error + let resolvedUrl = rawUrl ?? config?.url; + if (!resolvedUrl && curatedPages && curatedPages.length > 0) { + const firstEntry = curatedPages[0]; + const firstUrl = typeof firstEntry === 'string' ? firstEntry : firstEntry.url; + resolvedUrl = new URL(firstUrl).origin; + } if (!resolvedUrl) { process.stderr.write( 'Error: No URL provided. Pass a URL as an argument or set "url" in agent-docs.config.yml\n', @@ -64,8 +112,6 @@ export function registerCheckCommand(program: Command): void { return; } - const samplingRaw = - (opts.sampling as string | undefined) ?? config?.options?.samplingStrategy ?? 'random'; const sampling = samplingRaw as SamplingStrategy; if (!SAMPLING_STRATEGIES.includes(sampling)) { process.stderr.write( @@ -75,6 +121,14 @@ export function registerCheckCommand(program: Command): void { return; } + if (sampling === 'curated' && (!curatedPages || curatedPages.length === 0)) { + process.stderr.write( + 'Error: Curated sampling requires pages. Use --urls or define "pages" in your config file.\n', + ); + process.exitCode = 1; + return; + } + const maxConcurrency = parseInt( String((opts.maxConcurrency as string | undefined) ?? config?.options?.maxConcurrency ?? 3), 10, @@ -115,6 +169,7 @@ export function registerCheckCommand(program: Command): void { requestDelay, maxLinksToTest, samplingStrategy: sampling, + curatedPages, thresholds: { pass: passThreshold, fail: failThreshold, diff --git a/src/cli/formatters/scorecard.ts b/src/cli/formatters/scorecard.ts index 98d43f7..385824f 100644 --- a/src/cli/formatters/scorecard.ts +++ b/src/cli/formatters/scorecard.ts @@ -104,6 +104,36 @@ export function formatScorecard(report: ReportResult, scoreResult?: ScoreResult) } lines.push(''); + // Tag scores (when curated pages have tags) + if (score.tagScores) { + lines.push(` ${chalk.bold('Tag Scores:')}`); + const sortedTags = Object.entries(score.tagScores).sort(([a], [b]) => a.localeCompare(b)); + for (const [tag, tagScore] of sortedTags) { + lines.push( + formatCategoryLine(tag, tagScore.score, tagScore.grade) + + chalk.dim(` · ${tagScore.pageCount} page${tagScore.pageCount !== 1 ? 's' : ''}`), + ); + + // Show checks that aren't fully passing + const issues = tagScore.checks.filter((c) => c.proportion < 1); + for (const check of issues) { + const counts = { pass: 0, warn: 0, fail: 0 }; + for (const p of check.pages) { + if (p.status in counts) counts[p.status as keyof typeof counts]++; + } + const parts: string[] = []; + if (counts.fail > 0) parts.push(chalk.red(`${counts.fail} fail`)); + if (counts.warn > 0) parts.push(chalk.yellow(`${counts.warn} warn`)); + if (counts.pass > 0) parts.push(chalk.green(`${counts.pass} pass`)); + const worstStatus = counts.fail > 0 ? 'fail' : 'warn'; + const label = STATUS_LABELS[worstStatus]; + const color = STATUS_COLORS[worstStatus]; + lines.push(` ${color(label)} ${check.checkId.padEnd(30)} ${parts.join(', ')}`); + } + } + lines.push(''); + } + // Interaction diagnostics if (score.diagnostics.length > 0) { lines.push(` ${chalk.bold('Interaction Diagnostics:')}`); diff --git a/src/helpers/config.ts b/src/helpers/config.ts index a013943..31afbf2 100644 --- a/src/helpers/config.ts +++ b/src/helpers/config.ts @@ -1,10 +1,44 @@ import { readFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { parse as parseYaml } from 'yaml'; -import type { AgentDocsConfig } from '../types.js'; +import type { AgentDocsConfig, PageConfigEntry } from '../types.js'; const CONFIG_FILENAMES = ['agent-docs.config.yml', 'agent-docs.config.yaml']; +/** + * Validate the `pages` field in a config file. + * Each entry must be a valid URL string or an object with a valid `url` and optional `tag`. + */ +function assertPagesArray(pages: unknown, source: string): asserts pages is unknown[] { + if (!Array.isArray(pages)) { + throw new Error(`${source}: "pages" must be an array of URLs or { url, tag? } objects`); + } +} + +export function validatePages(pages: unknown[], source: string): void { + for (let i = 0; i < pages.length; i++) { + const entry = pages[i] as PageConfigEntry; + if (typeof entry === 'string') { + try { + new URL(entry); + } catch { + throw new Error(`${source}: pages[${i}] is not a valid URL: ${entry}`); + } + } else if (typeof entry === 'object' && entry !== null && typeof entry.url === 'string') { + try { + new URL(entry.url); + } catch { + throw new Error(`${source}: pages[${i}].url is not a valid URL: ${entry.url}`); + } + if (entry.tag !== undefined && typeof entry.tag !== 'string') { + throw new Error(`${source}: pages[${i}].tag must be a string`); + } + } else { + throw new Error(`${source}: pages[${i}] must be a URL string or { url, tag? } object`); + } + } +} + /** * Search for an agent-docs config file starting from `dir` and walking up * to the filesystem root (like eslint, prettier, etc.). @@ -22,6 +56,10 @@ export async function loadConfig(dir?: string): Promise { if (!parsed.url) { throw new Error(`Config file ${filepath} is missing required "url" field`); } + if (parsed.pages) { + assertPagesArray(parsed.pages, filepath); + validatePages(parsed.pages, filepath); + } return parsed; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue; @@ -53,7 +91,12 @@ export async function findConfig( if (explicitPath) { const filepath = resolve(process.cwd(), explicitPath); const content = await readFile(filepath, 'utf-8'); - return parseYaml(content) as AgentDocsConfig; + const parsed = parseYaml(content) as AgentDocsConfig; + if (parsed.pages) { + assertPagesArray(parsed.pages, filepath); + validatePages(parsed.pages, filepath); + } + return parsed; } let searchDir = resolve(startDir ?? process.cwd()); @@ -62,7 +105,12 @@ export async function findConfig( const filepath = resolve(searchDir, filename); try { const content = await readFile(filepath, 'utf-8'); - return parseYaml(content) as AgentDocsConfig; + const parsed = parseYaml(content) as AgentDocsConfig; + if (parsed.pages) { + assertPagesArray(parsed.pages, filepath); + validatePages(parsed.pages, filepath); + } + return parsed; } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue; throw err; diff --git a/src/helpers/get-page-urls.ts b/src/helpers/get-page-urls.ts index f19081d..6ead14a 100644 --- a/src/helpers/get-page-urls.ts +++ b/src/helpers/get-page-urls.ts @@ -343,6 +343,8 @@ export interface SampledPages { totalPages: number; sampled: boolean; warnings: string[]; + /** When curated pages have tags, maps page URL to tag label. */ + urlTags?: Record; } /** @@ -364,6 +366,42 @@ export async function discoverAndSamplePages(ctx: CheckContext): Promise = {}; + for (const entry of entries) { + if (typeof entry === 'string') { + urls.push(entry); + } else { + urls.push(entry.url); + if (entry.tag) { + urlTags[entry.url] = entry.tag; + } + } + } + + ctx._sampledPages = { + urls, + totalPages: urls.length, + sampled: false, + warnings: [], + urlTags: Object.keys(urlTags).length > 0 ? urlTags : undefined, + }; + return ctx._sampledPages; + } + // "none" skips discovery and uses only the URL the user provided. if (strategy === 'none') { ctx._sampledPages = { diff --git a/src/helpers/vitest-runner.ts b/src/helpers/vitest-runner.ts index 574809a..e4b8b41 100644 --- a/src/helpers/vitest-runner.ts +++ b/src/helpers/vitest-runner.ts @@ -48,9 +48,15 @@ export function describeAgentDocs( async () => { const config = await resolveConfig(configOrDir); + const inferredStrategy = + config.pages && config.pages.length > 0 && !config.options?.samplingStrategy + ? 'curated' + : undefined; const report = await runChecks(config.url, { checkIds: config.checks, ...config.options, + ...(inferredStrategy && { samplingStrategy: inferredStrategy as 'curated' }), + curatedPages: config.pages, }); results = report.results; }, @@ -92,9 +98,15 @@ export function describeAgentDocsPerCheck( // Run all checks once upfront beforeAll(async () => { const config = await resolveConfig(configOrDir); + const inferredStrategy = + config.pages && config.pages.length > 0 && !config.options?.samplingStrategy + ? 'curated' + : undefined; report = await runChecks(config.url, { checkIds: config.checks, ...config.options, + ...(inferredStrategy && { samplingStrategy: inferredStrategy as 'curated' }), + curatedPages: config.pages, }); resultsByCheck = new Map(report.results.map((r) => [r.id, r])); }, timeout); diff --git a/src/index.ts b/src/index.ts index 6a05cb9..4209c75 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,8 @@ export type { AgentDocsConfig, DiscoveredFile, SizeThresholds, + CuratedPageEntry, + PageConfigEntry, } from './types.js'; export { DEFAULT_OPTIONS, DEFAULT_THRESHOLDS, CATEGORIES } from './constants.js'; @@ -24,6 +26,8 @@ export type { ScoreResult, CheckScore, CategoryScore, + TagScore, + TagCheckBreakdown, ScoreCap, Diagnostic, DiagnosticSeverity, diff --git a/src/runner.ts b/src/runner.ts index b57f1a9..d8a509b 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -57,6 +57,7 @@ export function createContext(baseUrl: string, options?: Partial) options: merged, pageCache: new Map(), htmlCache: new Map(), + _curatedPages: options?.curatedPages, }; } @@ -123,11 +124,14 @@ export async function runChecks( error: results.filter((r) => r.status === 'error').length, }; + const urlTags = ctx._sampledPages?.urlTags; + return { url: baseUrl, timestamp: new Date().toISOString(), specUrl: SPEC_BASE_URL, results, summary, + ...(urlTags && { urlTags }), }; } diff --git a/src/scoring/index.ts b/src/scoring/index.ts index 6f54b90..ce78052 100644 --- a/src/scoring/index.ts +++ b/src/scoring/index.ts @@ -5,11 +5,14 @@ export { evaluateDiagnostics } from './diagnostics.js'; export { getResolution } from './resolutions.js'; export { getCheckProportion } from './proportions.js'; export { getCoefficient } from './coefficients.js'; +export { computeTagScores } from './tag-scores.js'; export type { ScoreResult, CheckScore, CategoryScore, + TagScore, + TagCheckBreakdown, ScoreCap, Diagnostic, DiagnosticSeverity, diff --git a/src/scoring/score.ts b/src/scoring/score.ts index f74df09..8d2c90e 100644 --- a/src/scoring/score.ts +++ b/src/scoring/score.ts @@ -6,6 +6,7 @@ import { getCheckProportion } from './proportions.js'; import { getCoefficient } from './coefficients.js'; import { evaluateDiagnostics } from './diagnostics.js'; import { getResolution } from './resolutions.js'; +import { computeTagScores } from './tag-scores.js'; /** * Compute a score from a report result. @@ -108,6 +109,11 @@ export function computeScore(report: ReportResult): ScoreResult { scoreResult.cap = cap; } + const tagScores = computeTagScores(report, checkScores); + if (tagScores) { + scoreResult.tagScores = tagScores; + } + return scoreResult; } diff --git a/src/scoring/tag-scores.ts b/src/scoring/tag-scores.ts new file mode 100644 index 0000000..86cd71a --- /dev/null +++ b/src/scoring/tag-scores.ts @@ -0,0 +1,220 @@ +import type { ReportResult } from '../types.js'; +import type { CheckScore, TagCheckBreakdown, TagScore } from './types.js'; +import { toGrade } from './score.js'; +import { CHECK_WEIGHTS } from './weights.js'; + +/** + * Extract the page-results array and a status normalizer for a check. + * + * Returns undefined for single-resource checks (no per-page data). + */ +function getPageItems( + details: Record, + checkId: string, +): Array<{ url: string; status: string }> | undefined { + // Known page-array field names and their status mapping + const arrayField = PAGE_ARRAY_FIELDS[checkId] ?? 'pageResults'; + const arr = details[arrayField] as Array> | undefined; + if (!Array.isArray(arr) || arr.length === 0) return undefined; + + const statusMapper = STATUS_MAPPERS[checkId] ?? defaultStatusMapper; + + return arr + .filter((item) => typeof item.url === 'string') + .map((item) => ({ + url: item.url as string, + status: statusMapper(item), + })); +} + +function defaultStatusMapper(item: Record): string { + if (typeof item.status === 'string') return item.status; + return 'skip'; +} + +// Checks that use a non-standard field name for their page array +const PAGE_ARRAY_FIELDS: Record = { + 'tabbed-content-serialization': 'tabbedPages', + 'section-header-quality': 'analyses', + 'cache-header-hygiene': 'endpointResults', +}; + +// Checks that don't store a `status` string directly and need custom mapping +const STATUS_MAPPERS: Record) => string> = { + 'content-negotiation': (item) => { + switch (item.classification) { + case 'markdown-with-correct-type': + return 'pass'; + case 'markdown-with-wrong-type': + return 'warn'; + case 'html': + return 'fail'; + default: + return 'skip'; + } + }, + + 'markdown-url-support': (item) => { + if (item.skipped) return 'skip'; + return item.supported ? 'pass' : 'fail'; + }, + + 'http-status-codes': (item) => { + switch (item.classification) { + case 'correct-error': + return 'pass'; + case 'soft-404': + return 'fail'; + default: + return 'skip'; + } + }, + + 'redirect-behavior': (item) => { + switch (item.classification) { + case 'no-redirect': + case 'same-host': + return 'pass'; + case 'cross-host': + return 'warn'; + case 'js-redirect': + return 'fail'; + default: + return 'skip'; + } + }, + + 'auth-gate-detection': (item) => { + switch (item.classification) { + case 'accessible': + return 'pass'; + case 'soft-auth-gate': + return 'warn'; + case 'auth-required': + case 'auth-redirect': + return 'fail'; + default: + return 'skip'; + } + }, + + 'llms-txt-directive': (item) => { + if (item.error) return 'skip'; + if (!item.found) return 'fail'; + if (typeof item.positionPercent === 'number' && item.positionPercent > 50) return 'warn'; + return 'pass'; + }, + + 'section-header-quality': (item) => { + if (item.hasGenericMajority) return 'fail'; + if (item.hasCrossGroupGeneric) return 'warn'; + return 'pass'; + }, +}; + +// Single-resource checks that should be excluded from tag scoring +const SINGLE_RESOURCE_CHECKS = new Set([ + 'llms-txt-exists', + 'llms-txt-valid', + 'llms-txt-size', + 'llms-txt-links-resolve', + 'llms-txt-links-markdown', + 'llms-txt-freshness', +]); + +/** + * Compute per-tag aggregate scores from curated page tags. + * + * For each tag, walks all per-page check results, filters to the tag's URLs, + * and computes a weighted aggregate score. + * + * Returns undefined when no tags are present. + */ +export function computeTagScores( + report: ReportResult, + checkScores: Record, +): Record | undefined { + const urlTags = report.urlTags; + if (!urlTags || Object.keys(urlTags).length === 0) return undefined; + + // Build tag -> URL set + const tagUrls = new Map>(); + for (const [url, tag] of Object.entries(urlTags)) { + let urls = tagUrls.get(tag); + if (!urls) { + urls = new Set(); + tagUrls.set(tag, urls); + } + urls.add(url); + } + + const tagScores: Record = {}; + + for (const [tag, urls] of tagUrls) { + let earned = 0; + let max = 0; + const checks: TagCheckBreakdown[] = []; + + for (const result of report.results) { + if (SINGLE_RESOURCE_CHECKS.has(result.id)) continue; + if (!result.details) continue; + + const cs = checkScores[result.id]; + if (!cs) continue; + + const items = getPageItems(result.details, result.id); + if (!items) continue; + + // Filter to this tag's URLs + const tagItems = items.filter((item) => urls.has(item.url)); + if (tagItems.length === 0) continue; + + let pass = 0; + let warn = 0; + let total = 0; + + for (const item of tagItems) { + if (item.status === 'pass') { + pass++; + total++; + } else if (item.status === 'warn') { + warn++; + total++; + } else if (item.status === 'fail') { + total++; + } + // skip/error excluded from proportion + } + + if (total === 0) continue; + + const warnCoeff = CHECK_WEIGHTS[result.id]?.warnCoefficient ?? 0.5; + const proportion = (pass + warn * warnCoeff) / total; + earned += proportion * cs.effectiveWeight; + max += cs.effectiveWeight; + + checks.push({ + checkId: result.id, + category: result.category, + weight: cs.effectiveWeight, + proportion, + pages: tagItems, + }); + } + + // Skip tags where no per-page checks contributed data. + // A score of 0 would imply "all checks failed," which is different from + // "no checks produced results for these URLs." + if (max === 0) continue; + + const score = Math.round((earned / max) * 100); + tagScores[tag] = { + score, + grade: toGrade(score), + pageCount: urls.size, + checks, + }; + } + + return Object.keys(tagScores).length > 0 ? tagScores : undefined; +} diff --git a/src/scoring/types.ts b/src/scoring/types.ts index b66ca24..2212b05 100644 --- a/src/scoring/types.ts +++ b/src/scoring/types.ts @@ -22,6 +22,21 @@ export interface CategoryScore { grade: Grade; } +export interface TagCheckBreakdown { + checkId: string; + category: string; + weight: number; + proportion: number; + pages: Array<{ url: string; status: string }>; +} + +export interface TagScore { + score: number; + grade: Grade; + pageCount: number; + checks: TagCheckBreakdown[]; +} + export interface ScoreCap { /** The cap value applied. */ cap: number; @@ -48,4 +63,6 @@ export interface ScoreResult { diagnostics: Diagnostic[]; /** Per-check resolution text for warn/fail checks, keyed by check ID. */ resolutions: Record; + /** Per-tag aggregate scores when curated pages have tags. */ + tagScores?: Record; } diff --git a/src/types.ts b/src/types.ts index 385bdcc..09fd40d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -51,9 +51,19 @@ export interface CheckContext { htmlCache: Map; /** Cached sampled pages result, shared across checks within a single run. */ _sampledPages?: SampledPages; + /** Curated page list from config or --urls, used by the curated sampling strategy. */ + _curatedPages?: PageConfigEntry[]; } -export type SamplingStrategy = 'random' | 'deterministic' | 'none'; +export type SamplingStrategy = 'random' | 'deterministic' | 'curated' | 'none'; + +export interface CuratedPageEntry { + url: string; + tag?: string; +} + +/** A page in the config `pages` array: either a bare URL string or an object with url + tag. */ +export type PageConfigEntry = string | CuratedPageEntry; export interface CheckOptions { /** Maximum concurrent HTTP requests within a single check. */ @@ -121,6 +131,8 @@ export interface DiscoveredFile { export interface RunnerOptions extends CheckOptions { /** Only run checks matching these IDs. If empty, run all. */ checkIds?: string[]; + /** Curated page list from config or --urls. Used when samplingStrategy is 'curated'. */ + curatedPages?: PageConfigEntry[]; } export interface ReportResult { @@ -136,10 +148,14 @@ export interface ReportResult { skip: number; error: number; }; + /** When curated pages have tags, maps page URL to tag label. */ + urlTags?: Record; } export interface AgentDocsConfig { url: string; checks?: string[]; options?: Partial; + /** Curated page URLs to test. Implies `samplingStrategy: 'curated'` when no strategy is set. */ + pages?: PageConfigEntry[]; } diff --git a/test/unit/cli/check-command.test.ts b/test/unit/cli/check-command.test.ts index dee50f8..ebc4281 100644 --- a/test/unit/cli/check-command.test.ts +++ b/test/unit/cli/check-command.test.ts @@ -366,6 +366,30 @@ describe('check command config integration', () => { writeSpy.mockRestore(); }); + it('errors when --sampling curated is used without pages', async () => { + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://no-pages.local', + '--sampling', + 'curated', + '--request-delay', + '0', + ]); + + await new Promise((r) => setTimeout(r, 100)); + + const output = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(output).toContain('Curated sampling requires pages'); + expect(process.exitCode).toBe(1); + + stderrSpy.mockRestore(); + }); + it('--checks flag overrides config checks', async () => { server.use( http.get('http://cfg-checks-override.local/llms.txt', () => @@ -406,4 +430,148 @@ describe('check command config integration', () => { writeSpy.mockRestore(); }); + + it('infers base URL from --urls when no URL argument given', async () => { + server.use( + http.get('http://infer-url.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get( + 'http://infer-url.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + '--urls', + 'http://infer-url.local/a', + '--checks', + 'llms-txt-exists', + '--format', + 'json', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const output = writeSpy.mock.calls.map((c) => c[0]).join(''); + const parsed = JSON.parse(output.trim()); + expect(parsed.url).toBe('http://infer-url.local'); + + writeSpy.mockRestore(); + }); + + it('errors when --urls value is empty', async () => { + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://example.local', + '--urls', + ' , , ', + '--request-delay', + '0', + ]); + + await new Promise((r) => setTimeout(r, 100)); + + const output = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(output).toContain('--urls requires at least one URL'); + expect(process.exitCode).toBe(1); + + stderrSpy.mockRestore(); + }); + + it('rejects invalid sampling strategy', async () => { + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://example.local', + '--sampling', + 'invalid-strategy', + '--request-delay', + '0', + ]); + + await new Promise((r) => setTimeout(r, 100)); + + const output = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(output).toContain('Invalid sampling strategy'); + expect(output).toContain('invalid-strategy'); + expect(process.exitCode).toBe(1); + + stderrSpy.mockRestore(); + }); + + it('rejects invalid URLs in --urls', async () => { + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://example.local', + '--urls', + 'not-a-url', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const output = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(output).toContain('not a valid URL'); + expect(process.exitCode).toBe(1); + + stderrSpy.mockRestore(); + }); + + it('infers base URL from config pages when url field is omitted', async () => { + server.use( + http.get('http://cfg-infer.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get( + 'http://cfg-infer.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const configPath = resolve(CONFIG_TMP, 'agent-docs.config.yml'); + await writeFile( + configPath, + 'pages:\n - http://cfg-infer.local/a\nchecks:\n - llms-txt-exists\n', + ); + + const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + '--config', + configPath, + '--format', + 'json', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const output = writeSpy.mock.calls.map((c) => c[0]).join(''); + const parsed = JSON.parse(output.trim()); + expect(parsed.url).toBe('http://cfg-infer.local'); + + writeSpy.mockRestore(); + }); }); diff --git a/test/unit/cli/scorecard-formatter.test.ts b/test/unit/cli/scorecard-formatter.test.ts index 542fc50..311fd1b 100644 --- a/test/unit/cli/scorecard-formatter.test.ts +++ b/test/unit/cli/scorecard-formatter.test.ts @@ -367,4 +367,49 @@ describe('formatScorecard', () => { expect(output).toContain('(D)'); expect(output).toContain('(F)'); }); + + it('shows tag scores when present', () => { + const score = makeScoreResult({ + tagScores: { + 'getting-started': { score: 90, grade: 'A', pageCount: 3, checks: [] }, + 'api-reference': { + score: 65, + grade: 'D', + pageCount: 5, + checks: [ + { + checkId: 'page-size-html', + category: 'page-size', + weight: 7, + proportion: 0.6, + pages: [ + { url: 'https://example.com/a', status: 'pass' }, + { url: 'https://example.com/b', status: 'fail' }, + { url: 'https://example.com/c', status: 'warn' }, + ], + }, + ], + }, + }, + }); + const output = formatScorecard(makeReport(), score); + expect(output).toContain('Tag Scores:'); + expect(output).toContain('api-reference'); + expect(output).toContain('65 / 100'); + expect(output).toContain('getting-started'); + expect(output).toContain('90 / 100'); + // Shows page counts + expect(output).toContain('3 pages'); + expect(output).toContain('5 pages'); + // Shows check breakdown for non-passing checks + expect(output).toContain('page-size-html'); + expect(output).toContain('1 fail'); + expect(output).toContain('1 warn'); + expect(output).toContain('1 pass'); + }); + + it('omits tag scores section when not present', () => { + const output = formatScorecard(makeReport(), makeScoreResult()); + expect(output).not.toContain('Tag Scores:'); + }); }); diff --git a/test/unit/helpers/config.test.ts b/test/unit/helpers/config.test.ts index 161d8c3..c12f2a9 100644 --- a/test/unit/helpers/config.test.ts +++ b/test/unit/helpers/config.test.ts @@ -65,6 +65,37 @@ describe('loadConfig', () => { expect(config.url).toBe('https://parent.example.com'); }); + it('validates pages when present', async () => { + await mkdir(TMP_DIR, { recursive: true }); + await writeFile( + resolve(TMP_DIR, 'agent-docs.config.yml'), + 'url: https://example.com\npages:\n - https://example.com/a\n - https://example.com/b\n', + ); + + const config = await loadConfig(TMP_DIR); + expect(config.pages).toEqual(['https://example.com/a', 'https://example.com/b']); + }); + + it('throws on invalid pages in loadConfig', async () => { + await mkdir(TMP_DIR, { recursive: true }); + await writeFile( + resolve(TMP_DIR, 'agent-docs.config.yml'), + 'url: https://example.com\npages:\n - not-a-url\n', + ); + + await expect(loadConfig(TMP_DIR)).rejects.toThrow('pages[0] is not a valid URL'); + }); + + it('throws on scalar pages in loadConfig', async () => { + await mkdir(TMP_DIR, { recursive: true }); + await writeFile( + resolve(TMP_DIR, 'agent-docs.config.yml'), + 'url: https://example.com\npages: https://example.com/a\n', + ); + + await expect(loadConfig(TMP_DIR)).rejects.toThrow('"pages" must be an array'); + }); + it('finds config in immediate dir before walking up', async () => { const parentDir = TMP_DIR; const childDir = resolve(TMP_DIR, 'sub'); @@ -145,4 +176,82 @@ describe('findConfig', () => { const config = await findConfig(undefined, TMP_DIR); expect(config?.url).toBe('https://yml.example.com'); }); + + it('auto-discovers config with pages and validates them', async () => { + await mkdir(TMP_DIR, { recursive: true }); + await writeFile( + resolve(TMP_DIR, 'agent-docs.config.yml'), + 'url: https://example.com\npages:\n - https://example.com/a\n', + ); + + const config = await findConfig(undefined, TMP_DIR); + expect(config?.pages).toEqual(['https://example.com/a']); + }); + + it('auto-discover throws on invalid pages', async () => { + await mkdir(TMP_DIR, { recursive: true }); + await writeFile( + resolve(TMP_DIR, 'agent-docs.config.yml'), + 'url: https://example.com\npages:\n - not-a-url\n', + ); + + await expect(findConfig(undefined, TMP_DIR)).rejects.toThrow('pages[0] is not a valid URL'); + }); + + it('loads pages as string array', async () => { + await mkdir(TMP_DIR, { recursive: true }); + const configPath = resolve(TMP_DIR, 'with-pages.yml'); + await writeFile( + configPath, + 'url: https://example.com\npages:\n - https://example.com/a\n - https://example.com/b\n', + ); + + const config = await findConfig(configPath); + expect(config?.pages).toEqual(['https://example.com/a', 'https://example.com/b']); + }); + + it('loads pages as mixed string and object array', async () => { + await mkdir(TMP_DIR, { recursive: true }); + const configPath = resolve(TMP_DIR, 'mixed-pages.yml'); + await writeFile( + configPath, + [ + 'url: https://example.com', + 'pages:', + ' - https://example.com/a', + ' - url: https://example.com/b', + ' tag: api', + '', + ].join('\n'), + ); + + const config = await findConfig(configPath); + expect(config?.pages).toHaveLength(2); + expect(config?.pages?.[0]).toBe('https://example.com/a'); + expect(config?.pages?.[1]).toEqual({ url: 'https://example.com/b', tag: 'api' }); + }); + + it('throws on invalid URL in pages', async () => { + await mkdir(TMP_DIR, { recursive: true }); + const configPath = resolve(TMP_DIR, 'bad-pages.yml'); + await writeFile(configPath, 'url: https://example.com\npages:\n - not-a-url\n'); + + await expect(findConfig(configPath)).rejects.toThrow('pages[0] is not a valid URL'); + }); + + it('throws on invalid object entry in pages', async () => { + await mkdir(TMP_DIR, { recursive: true }); + const configPath = resolve(TMP_DIR, 'bad-obj.yml'); + await writeFile(configPath, 'url: https://example.com\npages:\n - foo: bar\n'); + + await expect(findConfig(configPath)).rejects.toThrow('pages[0] must be a URL string or'); + }); + + it('throws when pages is a scalar instead of an array', async () => { + await mkdir(TMP_DIR, { recursive: true }); + const configPath = resolve(TMP_DIR, 'scalar-pages.yml'); + await writeFile(configPath, 'url: https://example.com\npages: https://example.com/a\n'); + + await expect(findConfig(configPath)).rejects.toThrow('"pages" must be an array'); + }); }); diff --git a/test/unit/helpers/get-page-urls.test.ts b/test/unit/helpers/get-page-urls.test.ts index 24e0ebe..edb79b8 100644 --- a/test/unit/helpers/get-page-urls.test.ts +++ b/test/unit/helpers/get-page-urls.test.ts @@ -742,6 +742,66 @@ describe('discoverAndSamplePages', () => { expect(result.sampled).toBe(false); }); + it('curated strategy returns configured URLs without discovery', async () => { + const ctx = createContext('http://curated.local', { + requestDelay: 0, + samplingStrategy: 'curated', + }); + ctx._curatedPages = ['http://curated.local/page-a', 'http://curated.local/page-b']; + + const result = await discoverAndSamplePages(ctx); + expect(result.urls).toEqual(['http://curated.local/page-a', 'http://curated.local/page-b']); + expect(result.totalPages).toBe(2); + expect(result.sampled).toBe(false); + expect(result.urlTags).toBeUndefined(); + }); + + it('curated strategy with tagged objects populates urlTags', async () => { + const ctx = createContext('http://curated-tags.local', { + requestDelay: 0, + samplingStrategy: 'curated', + }); + ctx._curatedPages = [ + 'http://curated-tags.local/page-a', + { url: 'http://curated-tags.local/page-b', tag: 'api' }, + { url: 'http://curated-tags.local/page-c', tag: 'guides' }, + ]; + + const result = await discoverAndSamplePages(ctx); + expect(result.urls).toHaveLength(3); + expect(result.urlTags).toEqual({ + 'http://curated-tags.local/page-b': 'api', + 'http://curated-tags.local/page-c': 'guides', + }); + }); + + it('curated strategy with empty pages falls back to baseUrl', async () => { + const ctx = createContext('http://curated-empty.local', { + requestDelay: 0, + samplingStrategy: 'curated', + }); + ctx._curatedPages = []; + + const result = await discoverAndSamplePages(ctx); + expect(result.urls).toEqual(['http://curated-empty.local']); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain('no pages defined'); + }); + + it('curated strategy does not apply maxLinksToTest', async () => { + const urls = Array.from({ length: 100 }, (_, i) => `http://curated-many.local/page-${i}`); + const ctx = createContext('http://curated-many.local', { + requestDelay: 0, + samplingStrategy: 'curated', + maxLinksToTest: 5, + }); + ctx._curatedPages = urls; + + const result = await discoverAndSamplePages(ctx); + expect(result.urls).toHaveLength(100); + expect(result.sampled).toBe(false); + }); + it('passes through warnings from discovery', async () => { server.use( http.get( diff --git a/test/unit/runner.test.ts b/test/unit/runner.test.ts index f51534e..b687eeb 100644 --- a/test/unit/runner.test.ts +++ b/test/unit/runner.test.ts @@ -62,6 +62,17 @@ describe('createContext URL normalization', () => { const ctx = createContext('example.com/'); expect(ctx.baseUrl).toBe('https://example.com'); }); + + it('stores curatedPages on _curatedPages', () => { + const pages = ['https://example.com/a', { url: 'https://example.com/b', tag: 'api' }]; + const ctx = createContext('https://example.com', { curatedPages: pages }); + expect(ctx._curatedPages).toEqual(pages); + }); + + it('leaves _curatedPages undefined when not provided', () => { + const ctx = createContext('https://example.com'); + expect(ctx._curatedPages).toBeUndefined(); + }); }); describe('runner', () => { diff --git a/test/unit/scoring/score.test.ts b/test/unit/scoring/score.test.ts index 737d001..af4ea6a 100644 --- a/test/unit/scoring/score.test.ts +++ b/test/unit/scoring/score.test.ts @@ -409,4 +409,43 @@ describe('computeScore', () => { expect(score.resolutions['markdown-url-support']).toBeDefined(); }); }); + + describe('tag scores integration', () => { + it('includes tagScores when report has urlTags', () => { + const results: CheckResult[] = [ + makeResult('page-size-html', 'page-size', 'warn', { + passBucket: 1, + warnBucket: 1, + failBucket: 0, + pageResults: [ + { url: 'https://example.com/a', status: 'pass' }, + { url: 'https://example.com/b', status: 'warn' }, + ], + }), + ]; + const report = makeReport(results); + report.urlTags = { + 'https://example.com/a': 'docs', + 'https://example.com/b': 'api', + }; + + const score = computeScore(report); + expect(score.tagScores).toBeDefined(); + expect(score.tagScores!['docs'].score).toBe(100); + expect(score.tagScores!['api'].score).toBe(50); // warn maps to 0.5 + }); + + it('omits tagScores when report has no urlTags', () => { + const results: CheckResult[] = [ + makeResult('page-size-html', 'page-size', 'pass', { + passBucket: 1, + warnBucket: 0, + failBucket: 0, + pageResults: [{ url: 'https://example.com/a', status: 'pass' }], + }), + ]; + const score = computeScore(makeReport(results)); + expect(score.tagScores).toBeUndefined(); + }); + }); }); diff --git a/test/unit/scoring/tag-scores.test.ts b/test/unit/scoring/tag-scores.test.ts new file mode 100644 index 0000000..fa558da --- /dev/null +++ b/test/unit/scoring/tag-scores.test.ts @@ -0,0 +1,500 @@ +import { describe, it, expect } from 'vitest'; +import { computeTagScores } from '../../../src/scoring/tag-scores.js'; +import type { CheckResult, ReportResult } from '../../../src/types.js'; +import type { CheckScore } from '../../../src/scoring/types.js'; + +function makeResult( + id: string, + category: string, + status: CheckResult['status'], + details?: Record, +): CheckResult { + return { id, category, status, message: `${id}: ${status}`, details }; +} + +function makeReport(results: CheckResult[], urlTags?: Record): ReportResult { + const summary = { total: results.length, pass: 0, warn: 0, fail: 0, skip: 0, error: 0 }; + for (const r of results) { + summary[r.status]++; + } + return { + url: 'https://example.com', + timestamp: new Date().toISOString(), + specUrl: 'https://agentdocsspec.com/spec/', + results, + summary, + urlTags, + }; +} + +function makeCheckScore(effectiveWeight: number): CheckScore { + return { + baseWeight: effectiveWeight, + coefficient: 1, + effectiveWeight, + proportion: 1, + earnedScore: effectiveWeight, + maxScore: effectiveWeight, + }; +} + +describe('computeTagScores', () => { + it('returns undefined when no urlTags', () => { + const report = makeReport([]); + const result = computeTagScores(report, {}); + expect(result).toBeUndefined(); + }); + + it('returns undefined when urlTags is empty', () => { + const report = makeReport([], {}); + const result = computeTagScores(report, {}); + expect(result).toBeUndefined(); + }); + + it('computes tag scores from pageResults with status field', () => { + const report = makeReport( + [ + makeResult('page-size-html', 'page-size', 'warn', { + passBucket: 2, + warnBucket: 1, + failBucket: 0, + pageResults: [ + { url: 'https://example.com/a', status: 'pass' }, + { url: 'https://example.com/b', status: 'pass' }, + { url: 'https://example.com/c', status: 'warn' }, + ], + }), + ], + { + 'https://example.com/a': 'getting-started', + 'https://example.com/b': 'api', + 'https://example.com/c': 'api', + }, + ); + + const checkScores: Record = { + 'page-size-html': makeCheckScore(7), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + + // getting-started: 1 page passing -> proportion 1.0 -> score 100 + expect(result!['getting-started'].score).toBe(100); + expect(result!['getting-started'].grade).toBe('A+'); + expect(result!['getting-started'].pageCount).toBe(1); + expect(result!['getting-started'].checks).toHaveLength(1); + expect(result!['getting-started'].checks[0].checkId).toBe('page-size-html'); + expect(result!['getting-started'].checks[0].proportion).toBe(1); + expect(result!['getting-started'].checks[0].pages).toEqual([ + { url: 'https://example.com/a', status: 'pass' }, + ]); + + // api: 1 pass + 1 warn -> proportion (1 + 0.5) / 2 = 0.75 -> score 75 + expect(result!['api'].score).toBe(75); + expect(result!['api'].grade).toBe('C'); + expect(result!['api'].pageCount).toBe(2); + expect(result!['api'].checks).toHaveLength(1); + expect(result!['api'].checks[0].proportion).toBe(0.75); + expect(result!['api'].checks[0].weight).toBe(7); + expect(result!['api'].checks[0].pages).toEqual([ + { url: 'https://example.com/b', status: 'pass' }, + { url: 'https://example.com/c', status: 'warn' }, + ]); + }); + + it('computes tag scores from classification-based checks', () => { + const report = makeReport( + [ + makeResult('auth-gate-detection', 'authentication', 'fail', { + pageResults: [ + { url: 'https://example.com/public', classification: 'accessible' }, + { url: 'https://example.com/private', classification: 'auth-required' }, + ], + }), + ], + { + 'https://example.com/public': 'docs', + 'https://example.com/private': 'docs', + }, + ); + + const checkScores: Record = { + 'auth-gate-detection': makeCheckScore(10), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + // 1 pass + 1 fail out of 2 -> proportion 0.5 -> score 50 + expect(result!['docs'].score).toBe(50); + }); + + it('handles tabbedPages field for tabbed-content-serialization', () => { + const report = makeReport( + [ + makeResult('tabbed-content-serialization', 'content-structure', 'pass', { + tabbedPages: [ + { url: 'https://example.com/a', status: 'pass' }, + { url: 'https://example.com/b', status: 'fail' }, + ], + }), + ], + { + 'https://example.com/a': 'tag-a', + 'https://example.com/b': 'tag-b', + }, + ); + + const checkScores: Record = { + 'tabbed-content-serialization': makeCheckScore(4), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + expect(result!['tag-a'].score).toBe(100); + expect(result!['tag-b'].score).toBe(0); + }); + + it('handles analyses field for section-header-quality', () => { + const report = makeReport( + [ + makeResult('section-header-quality', 'content-structure', 'warn', { + analyses: [ + { + url: 'https://example.com/good', + hasGenericMajority: false, + hasCrossGroupGeneric: false, + }, + { + url: 'https://example.com/bad', + hasGenericMajority: true, + hasCrossGroupGeneric: false, + }, + ], + }), + ], + { + 'https://example.com/good': 'quality', + 'https://example.com/bad': 'quality', + }, + ); + + const checkScores: Record = { + 'section-header-quality': makeCheckScore(2), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + // 1 pass + 1 fail -> 50 + expect(result!['quality'].score).toBe(50); + }); + + it('skips single-resource checks like llms-txt-exists', () => { + const report = makeReport( + [ + makeResult('llms-txt-exists', 'content-discoverability', 'pass', { + discoveredFiles: [{ url: 'https://example.com/llms.txt' }], + }), + ], + { 'https://example.com/a': 'tag-a' }, + ); + + const checkScores: Record = { + 'llms-txt-exists': makeCheckScore(10), + }; + + // llms-txt-exists is a single-resource check. Since no per-page checks + // contributed data, the tag is omitted entirely (no data ≠ score 0). + const result = computeTagScores(report, checkScores); + expect(result).toBeUndefined(); + }); + + it('omits tags whose URLs appear in no check results', () => { + const report = makeReport( + [ + makeResult('page-size-html', 'page-size', 'pass', { + pageResults: [{ url: 'https://example.com/a', status: 'pass' }], + }), + ], + { + 'https://example.com/a': 'found', + 'https://example.com/missing': 'not-found', + }, + ); + + const checkScores: Record = { + 'page-size-html': makeCheckScore(7), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + expect(result!['found'].score).toBe(100); + // not-found tag's URL doesn't appear in any pageResults, so the tag + // is omitted (no data ≠ score 0). + expect(result!['not-found']).toBeUndefined(); + }); + + it('aggregates across multiple checks', () => { + const report = makeReport( + [ + makeResult('page-size-html', 'page-size', 'pass', { + pageResults: [{ url: 'https://example.com/a', status: 'pass' }], + }), + makeResult('rendering-strategy', 'authentication', 'fail', { + pageResults: [{ url: 'https://example.com/a', status: 'fail' }], + }), + ], + { 'https://example.com/a': 'mixed' }, + ); + + const checkScores: Record = { + 'page-size-html': makeCheckScore(7), + 'rendering-strategy': makeCheckScore(10), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + // page-size-html: proportion=1.0 * weight=7 = 7 earned, 7 max + // rendering-strategy: proportion=0.0 * weight=10 = 0 earned, 10 max + // total: 7/17 = 41.2% -> 41 + expect(result!['mixed'].score).toBe(41); + expect(result!['mixed'].checks).toHaveLength(2); + expect(result!['mixed'].checks[0]).toMatchObject({ + checkId: 'page-size-html', + proportion: 1, + weight: 7, + }); + expect(result!['mixed'].checks[1]).toMatchObject({ + checkId: 'rendering-strategy', + proportion: 0, + weight: 10, + }); + }); + + it('uses per-check warn coefficients instead of hardcoded 0.5', () => { + // redirect-behavior has warnCoefficient 0.6, content-negotiation has 0.75 + const report = makeReport( + [ + makeResult('redirect-behavior', 'url-stability', 'warn', { + pageResults: [{ url: 'https://example.com/a', classification: 'cross-host' }], + }), + makeResult('content-negotiation', 'markdown-availability', 'warn', { + pageResults: [ + { url: 'https://example.com/a', classification: 'markdown-with-wrong-type' }, + ], + }), + ], + { 'https://example.com/a': 'test-tag' }, + ); + + const checkScores: Record = { + 'redirect-behavior': makeCheckScore(4), + 'content-negotiation': makeCheckScore(4), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + + // redirect-behavior: 1 warn with coeff 0.6 -> proportion 0.6 + // content-negotiation: 1 warn with coeff 0.75 -> proportion 0.75 + // If hardcoded to 0.5, both would be 0.5 and score would be 50. + // With correct coefficients: (0.6*4 + 0.75*4) / (4+4) = 5.4/8 = 0.675 -> 68 + expect(result!['test-tag'].checks[0].proportion).toBe(0.6); + expect(result!['test-tag'].checks[1].proportion).toBe(0.75); + expect(result!['test-tag'].score).toBe(68); + }); + + it('maps status correctly for all custom status mappers', () => { + const report = makeReport( + [ + // markdown-url-support: supported -> pass, !supported -> fail, skipped -> skip + makeResult('markdown-url-support', 'markdown-availability', 'pass', { + pageResults: [ + { url: 'https://example.com/a', supported: true }, + { url: 'https://example.com/b', supported: false }, + { url: 'https://example.com/c', skipped: true }, + ], + }), + // http-status-codes: correct-error -> pass, soft-404 -> fail + makeResult('http-status-codes', 'url-stability', 'fail', { + pageResults: [ + { url: 'https://example.com/a', classification: 'correct-error' }, + { url: 'https://example.com/b', classification: 'soft-404' }, + ], + }), + // llms-txt-directive: found near top -> pass, found deep -> warn, not found -> fail + makeResult('llms-txt-directive', 'content-discoverability', 'warn', { + pageResults: [ + { url: 'https://example.com/a', found: true, positionPercent: 5 }, + { url: 'https://example.com/b', found: true, positionPercent: 80 }, + { url: 'https://example.com/c', found: false }, + ], + }), + // cache-header-hygiene: uses endpointResults field + makeResult('cache-header-hygiene', 'observability', 'pass', { + endpointResults: [ + { url: 'https://example.com/a', status: 'pass' }, + { url: 'https://example.com/b', status: 'fail' }, + ], + }), + ], + { + 'https://example.com/a': 'all', + 'https://example.com/b': 'all', + 'https://example.com/c': 'all', + }, + ); + + const checkScores: Record = { + 'markdown-url-support': makeCheckScore(7), + 'http-status-codes': makeCheckScore(7), + 'llms-txt-directive': makeCheckScore(7), + 'cache-header-hygiene': makeCheckScore(2), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + + const checks = result!['all'].checks; + // markdown-url-support: 1 pass, 1 fail (skipped excluded) -> 0.5 + const mdUrl = checks.find((c) => c.checkId === 'markdown-url-support')!; + expect(mdUrl.proportion).toBe(0.5); + expect(mdUrl.pages).toHaveLength(3); + expect(mdUrl.pages[2].status).toBe('skip'); + + // http-status-codes: 1 pass, 1 fail -> 0.5 + const httpStatus = checks.find((c) => c.checkId === 'http-status-codes')!; + expect(httpStatus.proportion).toBe(0.5); + + // llms-txt-directive: 1 pass, 1 warn, 1 fail -> (1 + 0.6*1) / 3 + const directive = checks.find((c) => c.checkId === 'llms-txt-directive')!; + expect(directive.pages[0].status).toBe('pass'); + expect(directive.pages[1].status).toBe('warn'); + expect(directive.pages[2].status).toBe('fail'); + + // cache-header-hygiene: uses endpointResults, 1 pass + 1 fail -> 0.5 + const cache = checks.find((c) => c.checkId === 'cache-header-hygiene')!; + expect(cache.proportion).toBe(0.5); + }); + + it('covers remaining status mapper branches', () => { + const report = makeReport( + [ + // content-negotiation: pass, fail, and default/skip branches + makeResult('content-negotiation', 'markdown-availability', 'pass', { + pageResults: [ + { url: 'https://example.com/a', classification: 'markdown-with-correct-type' }, + { url: 'https://example.com/b', classification: 'html' }, + { url: 'https://example.com/c', classification: 'unknown-value' }, + ], + }), + // redirect-behavior: pass (no-redirect), fail (js-redirect), and default + makeResult('redirect-behavior', 'url-stability', 'warn', { + pageResults: [ + { url: 'https://example.com/a', classification: 'no-redirect' }, + { url: 'https://example.com/b', classification: 'js-redirect' }, + { url: 'https://example.com/c', classification: 'unknown-value' }, + ], + }), + // auth-gate-detection: soft-auth-gate (warn) and default (skip) + makeResult('auth-gate-detection', 'authentication', 'warn', { + pageResults: [ + { url: 'https://example.com/a', classification: 'soft-auth-gate' }, + { url: 'https://example.com/b', classification: 'unknown-value' }, + ], + }), + // http-status-codes: default/skip branch + makeResult('http-status-codes', 'url-stability', 'pass', { + pageResults: [{ url: 'https://example.com/a', classification: 'unknown-value' }], + }), + // llms-txt-directive: error -> skip + makeResult('llms-txt-directive', 'content-discoverability', 'pass', { + pageResults: [{ url: 'https://example.com/a', error: 'fetch failed' }], + }), + // section-header-quality: hasCrossGroupGeneric -> warn + makeResult('section-header-quality', 'content-structure', 'warn', { + analyses: [ + { url: 'https://example.com/a', hasGenericMajority: false, hasCrossGroupGeneric: true }, + ], + }), + ], + { + 'https://example.com/a': 'tag', + 'https://example.com/b': 'tag', + 'https://example.com/c': 'tag', + }, + ); + + const checkScores: Record = { + 'content-negotiation': makeCheckScore(4), + 'redirect-behavior': makeCheckScore(4), + 'auth-gate-detection': makeCheckScore(10), + 'http-status-codes': makeCheckScore(7), + 'llms-txt-directive': makeCheckScore(7), + 'section-header-quality': makeCheckScore(2), + }; + + const result = computeTagScores(report, checkScores); + expect(result).toBeDefined(); + const checks = result!['tag'].checks; + + // content-negotiation: pass + fail (skip excluded) -> 0.5 + const cn = checks.find((c) => c.checkId === 'content-negotiation')!; + expect(cn.pages[0].status).toBe('pass'); + expect(cn.pages[1].status).toBe('fail'); + expect(cn.pages[2].status).toBe('skip'); + expect(cn.proportion).toBe(0.5); + + // redirect-behavior: pass + fail (skip excluded) -> 0.5 + const rb = checks.find((c) => c.checkId === 'redirect-behavior')!; + expect(rb.pages[0].status).toBe('pass'); + expect(rb.pages[1].status).toBe('fail'); + expect(rb.pages[2].status).toBe('skip'); + + // auth-gate-detection: 1 warn (skip excluded) -> warnCoeff 0.5 + const ag = checks.find((c) => c.checkId === 'auth-gate-detection')!; + expect(ag.pages[0].status).toBe('warn'); + expect(ag.pages[1].status).toBe('skip'); + + // section-header-quality: hasCrossGroupGeneric -> warn + const shq = checks.find((c) => c.checkId === 'section-header-quality')!; + expect(shq.pages[0].status).toBe('warn'); + }); + + it('skips results with no details, no checkScore, empty pageResults, or all-skip items', () => { + const report = makeReport( + [ + // No details + makeResult('rendering-strategy', 'page-size', 'pass'), + // Has details but no matching checkScore + makeResult('page-size-html', 'page-size', 'pass', { + pageResults: [{ url: 'https://example.com/a', status: 'pass' }], + }), + // Has details but empty pageResults array + makeResult('content-start-position', 'page-size', 'pass', { + pageResults: [], + }), + // Has details but items have no string status (defaultStatusMapper -> skip) + makeResult('markdown-content-parity', 'observability', 'pass', { + pageResults: [{ url: 'https://example.com/a', result: { missing: 0 } }], + }), + ], + { 'https://example.com/a': 'tag' }, + ); + + // Only provide checkScore for some checks — page-size-html intentionally omitted + const checkScores: Record = { + 'rendering-strategy': makeCheckScore(10), + 'content-start-position': makeCheckScore(4), + 'markdown-content-parity': makeCheckScore(4), + }; + + // No checks produce scoreable data for the tag: + // - rendering-strategy: no details + // - page-size-html: no checkScore + // - content-start-position: empty pageResults + // - markdown-content-parity: all items map to 'skip', total=0 + const result = computeTagScores(report, checkScores); + expect(result).toBeUndefined(); + }); +});