Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
"@commitlint/config-conventional": "^20.0",
"@huggingface/transformers": "^3.8.1",
"@tree-sitter-grammars/tree-sitter-hcl": "^1.2.0",
"@types/better-sqlite3": "^7.6.13",
"@vitest/coverage-v8": "^4.0.18",
"commit-and-tag-version": "^12.5",
"husky": "^9.1",
Expand Down
1 change: 0 additions & 1 deletion src/cli/commands/info.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ export const command: CommandDefinition = {
const dbPath = findDbPath();
const fs = await import('node:fs');
if (fs.existsSync(dbPath)) {
// @ts-expect-error -- better-sqlite3 default export typing
const db = new Database(dbPath, { readonly: true });
const buildEngine = getBuildMeta(db, 'engine');
const buildVersion = getBuildMeta(db, 'codegraph_version');
Expand Down
4 changes: 2 additions & 2 deletions src/cli/shared/open-graph.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import type Database from 'better-sqlite3';
import { openReadonlyOrFail } from '../../db/index.js';
import type { BetterSqlite3Database } from '../../types.js';

/**
* Open the graph database in readonly mode with a clean close() handle.
*/
export function openGraph(opts: { db?: string } = {}): {
db: Database.Database;
db: BetterSqlite3Database;
close: () => void;
} {
const db = openReadonlyOrFail(opts.db);
Expand Down
65 changes: 52 additions & 13 deletions src/db/connection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,16 +143,12 @@ function isSameDirectory(a: string, b: string): boolean {
}

export function openDb(dbPath: string): LockedDatabase {
// Flush any deferred DB close from a previous build (avoids WAL contention)
flushDeferredClose();
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
acquireAdvisoryLock(dbPath);
// vendor.d.ts declares Database as a callable; cast through unknown for construct usage
const db = new (
Database as unknown as new (
path: string,
opts?: Record<string, unknown>,
) => LockedDatabase
)(dbPath);
const db = new Database(dbPath) as unknown as LockedDatabase;
db.pragma('journal_mode = WAL');
db.pragma('busy_timeout = 5000');
db.__lockPath = `${dbPath}.lock`;
Expand All @@ -164,6 +160,54 @@ export function closeDb(db: LockedDatabase): void {
if (db.__lockPath) releaseAdvisoryLock(db.__lockPath);
}

/** Pending deferred-close DB handles (not yet closed). */
const _deferredDbs: LockedDatabase[] = [];

/**
* Synchronously close any DB handles queued by `closeDbDeferred()`.
* Call before deleting DB files or in test teardown to avoid EBUSY on Windows.
*/
export function flushDeferredClose(): void {
while (_deferredDbs.length > 0) {
const db = _deferredDbs.pop()!;
try {
db.close();
} catch {
/* ignore — handle may already be closed */
}
}
}

/**
* Schedule DB close on the next event loop tick. Useful for incremental
* builds where the WAL checkpoint in db.close() is expensive (~250ms on
* Windows) and doesn't need to block the caller.
*
* The advisory lock is released immediately so subsequent opens succeed.
* The actual handle close (+ WAL checkpoint) happens asynchronously.
* Call `flushDeferredClose()` before deleting the DB file.
*/
export function closeDbDeferred(db: LockedDatabase): void {
// Release the advisory lock immediately so the next open can proceed
if (db.__lockPath) {
releaseAdvisoryLock(db.__lockPath);
db.__lockPath = undefined;
}
_deferredDbs.push(db);
// Defer the expensive WAL checkpoint to after the caller returns
setImmediate(() => {
const idx = _deferredDbs.indexOf(db);
if (idx !== -1) {
_deferredDbs.splice(idx, 1);
try {
db.close();
} catch {
/* ignore — handle may already be closed by flush */
}
}
});
}

export function findDbPath(customPath?: string): string {
if (customPath) return path.resolve(customPath);
const rawCeiling = findRepoRoot();
Expand Down Expand Up @@ -214,12 +258,7 @@ export function openReadonlyOrFail(customPath?: string): BetterSqlite3Database {
{ file: dbPath },
);
}
const db = new (
Database as unknown as new (
path: string,
opts?: Record<string, unknown>,
) => BetterSqlite3Database
)(dbPath, { readonly: true });
const db = new Database(dbPath, { readonly: true }) as unknown as BetterSqlite3Database;

// Warn once per process if the DB was built with a different codegraph version
if (!_versionWarned) {
Expand Down
2 changes: 2 additions & 0 deletions src/db/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
export type { LockedDatabase } from './connection.js';
export {
closeDb,
closeDbDeferred,
findDbPath,
findRepoRoot,
flushDeferredClose,
openDb,
openReadonlyOrFail,
openRepo,
Expand Down
4 changes: 2 additions & 2 deletions src/domain/graph/builder/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
* Each stage reads what it needs and writes what it produces.
* This replaces the closure-captured locals in the old monolithic buildGraph().
*/
import type BetterSqlite3 from 'better-sqlite3';
import type {
BetterSqlite3Database,
BuildGraphOpts,
CodegraphConfig,
EngineOpts,
Expand All @@ -20,7 +20,7 @@ import type {
export class PipelineContext {
// ── Inputs (set during setup) ──────────────────────────────────────
rootDir!: string;
db!: BetterSqlite3.Database;
db!: BetterSqlite3Database;
dbPath!: string;
config!: CodegraphConfig;
opts!: BuildGraphOpts;
Expand Down
26 changes: 15 additions & 11 deletions src/domain/graph/builder/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@
import { createHash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import type BetterSqlite3 from 'better-sqlite3';
import { purgeFilesData } from '../../../db/index.js';
import { debug, warn } from '../../../infrastructure/logger.js';
import { warn } from '../../../infrastructure/logger.js';
import { EXTENSIONS, IGNORE_DIRS } from '../../../shared/constants.js';
import type { BetterSqlite3Database, CodegraphConfig, PathAliases } from '../../../types.js';
import type {
BetterSqlite3Database,
CodegraphConfig,
PathAliases,
SqliteStatement,
} from '../../../types.js';

export const BUILTIN_RECEIVERS: Set<string> = new Set([
'console',
Expand Down Expand Up @@ -148,7 +152,7 @@ export function loadPathAliases(rootDir: string): PathAliases {
}
break;
} catch (err: unknown) {
debug(`Failed to parse ${configName}: ${(err as Error).message}`);
warn(`Failed to parse ${configName}: ${(err as Error).message}`);
}
}
return aliases;
Expand Down Expand Up @@ -198,7 +202,7 @@ export function readFileSafe(filePath: string, retries: number = 2): string {
* Purge all graph data for the specified files.
*/
export function purgeFilesFromGraph(
db: BetterSqlite3.Database,
db: BetterSqlite3Database,
files: string[],
options: Record<string, unknown> = {},
): void {
Expand All @@ -210,10 +214,10 @@ export function purgeFilesFromGraph(
const BATCH_CHUNK = 500;

// Statement caches keyed by chunk size — avoids recompiling for every batch.
const nodeStmtCache = new WeakMap<BetterSqlite3.Database, Map<number, BetterSqlite3.Statement>>();
const edgeStmtCache = new WeakMap<BetterSqlite3.Database, Map<number, BetterSqlite3.Statement>>();
const nodeStmtCache = new WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>();
const edgeStmtCache = new WeakMap<BetterSqlite3Database, Map<number, SqliteStatement>>();

function getNodeStmt(db: BetterSqlite3.Database, chunkSize: number): BetterSqlite3.Statement {
function getNodeStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatement {
let cache = nodeStmtCache.get(db);
if (!cache) {
cache = new Map();
Expand All @@ -231,7 +235,7 @@ function getNodeStmt(db: BetterSqlite3.Database, chunkSize: number): BetterSqlit
return stmt;
}

function getEdgeStmt(db: BetterSqlite3.Database, chunkSize: number): BetterSqlite3.Statement {
function getEdgeStmt(db: BetterSqlite3Database, chunkSize: number): SqliteStatement {
let cache = edgeStmtCache.get(db);
if (!cache) {
cache = new Map();
Expand All @@ -253,7 +257,7 @@ function getEdgeStmt(db: BetterSqlite3.Database, chunkSize: number): BetterSqlit
* Batch-insert node rows via multi-value INSERT statements.
* Each row: [name, kind, file, line, end_line, parent_id, qualified_name, scope, visibility]
*/
export function batchInsertNodes(db: BetterSqlite3.Database, rows: unknown[][]): void {
export function batchInsertNodes(db: BetterSqlite3Database, rows: unknown[][]): void {
if (!rows.length) return;
for (let i = 0; i < rows.length; i += BATCH_CHUNK) {
const end = Math.min(i + BATCH_CHUNK, rows.length);
Expand All @@ -272,7 +276,7 @@ export function batchInsertNodes(db: BetterSqlite3.Database, rows: unknown[][]):
* Batch-insert edge rows via multi-value INSERT statements.
* Each row: [source_id, target_id, kind, confidence, dynamic]
*/
export function batchInsertEdges(db: BetterSqlite3.Database, rows: unknown[][]): void {
export function batchInsertEdges(db: BetterSqlite3Database, rows: unknown[][]): void {
if (!rows.length) return;
for (let i = 0; i < rows.length; i += BATCH_CHUNK) {
const end = Math.min(i + BATCH_CHUNK, rows.length);
Expand Down
Loading
Loading