Improve data validation, fetch resilience, and database safety

package.json

@@ -5,7 +5,7 @@
   "description": "Workglow SEC is an example of using the Workglow AI library to build a tool for retrieving SEC data.",
   "scripts": {
     "bunset": "bunset --patch --push",
-    "publish": "bun run build && bun run test && bun run bunset",
+    "release": "bun run build && bun run test && bun run bunset",
     "dev": "concurrently -c 'auto' -n 'sec:' 'bun:dev-*'",
     "dev-js": "bun build --watch --target=bun --sourcemap=external --packages=external --outdir ./dist ./src/sec.ts",
     "dev-types": "tsc --watch --preserveWatchOutput",
@@ -15,7 +15,7 @@
     "build-types": "rm -f tsconfig.tsbuildinfo && tsc",
     "test": "bun test"
   },
-  "bin": "./src/sec.js",
+  "bin": "./dist/sec.js",
   "files": [
     "dist"
   ],
@@ -30,7 +30,6 @@
     "cheerio-json-mapper": "^1.0.4",
     "commander": "^14.0.3",
     "compromise": "^14.15.0",
-    "concurrently": "^9.2.1",
     "csv-parse": "^6.2.1",
     "fast-xml-parser": "^5.7.3",
     "html-entities": "^2.6.0",
@@ -44,7 +43,8 @@
     "@types/bun": "1.3.13",
     "@types/pg": "^8.15.2",
     "@types/xml2js": "^0.4.14",
-    "bunset": "1.0.12"
+    "bunset": "1.0.12",
+    "concurrently": "^9.2.1"
   },
   "trustedDependencies": [
     "@huggingface/transformers",

src/cli/GlobalOptions.test.ts

@@ -18,7 +18,6 @@ describe("GlobalOptions", () => {
       expect(help).toContain("--verbose");
       expect(help).toContain("--dry-run");
       expect(help).toContain("--no-color");
-      expect(help).toContain("--concurrency <n>");
     });
 
     it("returns the program for chaining", () => {
@@ -37,7 +36,6 @@ describe("GlobalOptions", () => {
       expect(opts.verbose).toBe(false);
       expect(opts.dryRun).toBe(false);
       expect(opts.color).toBe(true);
-      expect(opts.concurrency).toBeUndefined();
     });
   });
 
@@ -66,28 +64,16 @@ describe("GlobalOptions", () => {
       expect(parseGlobalOptions(program).color).toBe(false);
     });
 
-    it("parses --concurrency with a number", () => {
-      const program = createProgram();
-      program.parse(["--concurrency", "4"], { from: "user" });
-      expect(parseGlobalOptions(program).concurrency).toBe(4);
-    });
-
     it("parses all flags together", () => {
       const program = createProgram();
-      program.parse(["--json", "--verbose", "--dry-run", "--no-color", "--concurrency", "8"], {
+      program.parse(["--json", "--verbose", "--dry-run", "--no-color"], {
         from: "user",
       });
       const opts = parseGlobalOptions(program);
       expect(opts.json).toBe(true);
       expect(opts.verbose).toBe(true);
       expect(opts.dryRun).toBe(true);
       expect(opts.color).toBe(false);
-      expect(opts.concurrency).toBe(8);
-    });
-
-    it("throws on invalid concurrency value", () => {
-      const program = createProgram();
-      expect(() => program.parse(["--concurrency", "abc"], { from: "user" })).toThrow();
     });
   });
 });

src/cli/GlobalOptions.ts

@@ -5,16 +5,14 @@ export interface GlobalOptions {
   readonly verbose: boolean;
   readonly dryRun: boolean;
   readonly color: boolean;
-  readonly concurrency: number | undefined;
 }
 
 export function applyGlobalOptions(program: Command): Command {
   return program
     .option("--json", "Force JSON output", false)
     .option("--verbose", "Show detailed logs", false)
     .option("--dry-run", "Show what would happen without changes", false)
-    .option("--no-color", "Disable colored output")
-    .option("--concurrency <n>", "Override default concurrency", parseIntOption);
+    .option("--no-color", "Disable colored output");
 }
 
 export function parseGlobalOptions(cmd: Command): GlobalOptions {
@@ -24,7 +22,6 @@ export function parseGlobalOptions(cmd: Command): GlobalOptions {
     verbose: opts.verbose ?? false,
     dryRun: opts.dryRun ?? false,
     color: opts.color ?? true,
-    concurrency: opts.concurrency,
   };
 }
 

src/cli/cli.integration.test.ts

@@ -39,7 +39,6 @@ describe("CLI v2 integration", () => {
     expect(output).toContain("--verbose");
     expect(output).toContain("--dry-run");
     expect(output).toContain("--no-color");
-    expect(output).toContain("--concurrency");
   });
 
   it("should show version 2.0.0", async () => {

src/cli/groups/db.ts

@@ -1,8 +1,9 @@
 import type { Command } from "commander";
+import { resetAllDatabases } from "../../config/resetAllDatabases";
 import { setupAllDatabases } from "../../config/setupAllDatabases";
-import { runCommand } from "../runCommand";
-import { getDbStatus, getDbStats } from "../queries/DbStatus";
 import { renderTable } from "../output/TableRenderer";
+import { getDbStats, getDbStatus } from "../queries/DbStatus";
+import { runCommand } from "../runCommand";
 
 export function addDbCommands(program: Command): void {
   const db = program.command("db").description("Database management commands");
@@ -67,6 +68,10 @@ export function addDbCommands(program: Command): void {
         process.exitCode = 1;
         return;
       }
-      console.log("not yet implemented");
+      await runCommand(async () => {
+        await resetAllDatabases();
+        await setupAllDatabases();
+        console.log("Database reset complete.");
+      });
     });
 }

src/cli/groups/fetch.ts

@@ -13,6 +13,20 @@ import { secDate } from "../../util/parseDate";
 import { renderTable } from "../output/TableRenderer";
 import { runCommand } from "../runCommand";
 
+function parseCikArg(value: string): number {
+  // Require an all-digit string. parseInt would silently accept "123abc" as
+  // 123 and produce a plausible-looking but wrong CIK.
+  const trimmed = value.trim();
+  if (!/^\d+$/.test(trimmed)) {
+    throw new Error(`Invalid CIK "${value}": must be a positive integer`);
+  }
+  const parsed = Number(trimmed);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new Error(`Invalid CIK "${value}": must be a positive integer`);
+  }
+  return parsed;
+}
+
 async function listAvailableFormTypesForCik(cik: number): Promise<void> {
   const entityRepo = new EntityRepo();
   const filings = await entityRepo.getFilings(cik);
@@ -62,7 +76,7 @@ export function addFetchCommands(program: Command): void {
         wf.pipe(
           new FetchSubmissionsTask({
             defaults: {
-              cik: parseInt(cik),
+              cik: parseCikArg(cik),
               date: options.date ? secDate(options.date) : undefined,
             },
           }),
@@ -82,7 +96,7 @@ export function addFetchCommands(program: Command): void {
         wf.pipe(
           new FetchCompanyFactsTask({
             defaults: {
-              cik: parseInt(cik),
+              cik: parseCikArg(cik),
               date: options.date ? secDate(options.date) : undefined,
             },
           }),
@@ -99,7 +113,7 @@ export function addFetchCommands(program: Command): void {
     )
     .action(async (cik: string, form?: string, accession?: string) => {
       await runCommand(async () => {
-        const cikNum = parseInt(cik, 10);
+        const cikNum = parseCikArg(cik);
         if (form === undefined) {
           await listAvailableFormTypesForCik(cikNum);
           return;

src/commands/index.ts

@@ -5,7 +5,7 @@
  */
 
 import type { Command } from "commander";
-import { getTaskQueueRegistry, globalServiceRegistry } from "workglow";
+import { getTaskQueueRegistry, globalServiceRegistry, Sqlite } from "workglow";
 import { parseGlobalOptions } from "../cli/GlobalOptions";
 import { addBootstrapCommands } from "../cli/groups/bootstrap";
 import { addDbCommands } from "../cli/groups/db";
@@ -28,6 +28,13 @@ export const AddCommands = (program: Command): void => {
     if (diInitialized) return;
     diInitialized = true;
 
+    // Load the SQLite native binding only for commands that may open the DB,
+    // not for `init`, `--help`, `--version`, or any pure-CLI invocation.
+    const secDbType = process.env.SEC_DB_TYPE ?? "sqlite";
+    if (secDbType === "sqlite" && typeof Sqlite.init === "function") {
+      await Sqlite.init();
+    }
+
     const globalOpts = parseGlobalOptions(program);
     globalServiceRegistry.registerInstance(SEC_DRY_RUN, globalOpts.dryRun);
 

src/config/Constants.ts

@@ -4,5 +4,13 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-export const SecUserAgent = "PodleyAI SEC Job Queue <sroussey@gmail.com>";
+/**
+ * SEC fair-access policy requires a User-Agent in the form
+ *   "Sample Company Name AdminContact@samplecompany.com"
+ * EDGAR has been observed to 403 on RFC-5322 angle-bracket forms.
+ * Override at runtime via the SEC_USER_AGENT environment variable so each
+ * deployer identifies themselves rather than masquerading as the default.
+ */
+const DEFAULT_SEC_USER_AGENT = "PodleyAI SEC Job Queue sroussey@gmail.com";
+export const SecUserAgent = process.env.SEC_USER_AGENT?.trim() || DEFAULT_SEC_USER_AGENT;
 export const SecJobQueueName = "sec_job_queue";

src/config/DefaultDI.ts

@@ -5,6 +5,11 @@
  */
 
 import { globalServiceRegistry } from "workglow";
+import {
+  ADDRESS_HISTORY_JUNCTION_REPOSITORY_TOKEN,
+  AddressesEntityHistoryJunctionSchema,
+  AddressHistoryJunctionPrimaryKeyNames,
+} from "../storage/address/AddressHistorySchema";
 import {
   Address,
   ADDRESS_JUNCTION_REPOSITORY_TOKEN,
@@ -42,6 +47,11 @@ import {
   CikNamePrimaryKeyNames,
   CikNameSchema,
 } from "../storage/entity/CikNameSchema";
+import {
+  ENTITY_HISTORY_REPOSITORY_TOKEN,
+  EntityHistoryPrimaryKeyNames,
+  EntityHistorySchema,
+} from "../storage/entity/EntityHistorySchema";
 import {
   ENTITY_REPOSITORY_TOKEN,
   EntityPrimaryKeyNames,
@@ -197,6 +207,15 @@ export const DefaultDI = () => {
       ["cik"],
     ])
   );
+  globalServiceRegistry.registerInstance(
+    ADDRESS_HISTORY_JUNCTION_REPOSITORY_TOKEN,
+    createStorage(
+      "addresses_entity_history_junction",
+      AddressesEntityHistoryJunctionSchema,
+      AddressHistoryJunctionPrimaryKeyNames,
+      [["cik"]]
+    )
+  );
   // ------------------------------ Persons --------------------------------
   globalServiceRegistry.registerInstance(
     PERSON_REPOSITORY_TOKEN,
@@ -332,6 +351,12 @@ export const DefaultDI = () => {
     ENTITY_REPOSITORY_TOKEN,
     createStorage("entities", EntitySchema, EntityPrimaryKeyNames, [["name"], ["sic"]])
   );
+  globalServiceRegistry.registerInstance(
+    ENTITY_HISTORY_REPOSITORY_TOKEN,
+    createStorage("entities_history", EntityHistorySchema, EntityHistoryPrimaryKeyNames, [
+      ["valid_to"],
+    ])
+  );
   globalServiceRegistry.registerInstance(
     ENTITY_TICKER_REPOSITORY_TOKEN,
     createStorage("entity_tickers", EntityTickerSchema, EntityTickerPrimaryKeyNames, [

src/config/TestingDI.ts

@@ -5,6 +5,11 @@
  */
 
 import { InMemoryTabularStorage, globalServiceRegistry } from "workglow";
+import {
+  ADDRESS_HISTORY_JUNCTION_REPOSITORY_TOKEN,
+  AddressesEntityHistoryJunctionSchema,
+  AddressHistoryJunctionPrimaryKeyNames,
+} from "../storage/address/AddressHistorySchema";
 import {
   ADDRESS_JUNCTION_REPOSITORY_TOKEN,
   ADDRESS_REPOSITORY_TOKEN,
@@ -40,6 +45,11 @@ import {
   CikNamePrimaryKeyNames,
   CikNameSchema,
 } from "../storage/entity/CikNameSchema";
+import {
+  ENTITY_HISTORY_REPOSITORY_TOKEN,
+  EntityHistoryPrimaryKeyNames,
+  EntityHistorySchema,
+} from "../storage/entity/EntityHistorySchema";
 import {
   ENTITY_REPOSITORY_TOKEN,
   EntityPrimaryKeyNames,
@@ -255,6 +265,14 @@ export function resetDependencyInjectionsForTesting() {
       ["cik"],
     ])
   );
+  globalServiceRegistry.registerInstance(
+    ADDRESS_HISTORY_JUNCTION_REPOSITORY_TOKEN,
+    new InMemoryTabularStorage(
+      AddressesEntityHistoryJunctionSchema,
+      AddressHistoryJunctionPrimaryKeyNames,
+      [["cik"]]
+    )
+  );
 
   // Initialize Phone repositories
   globalServiceRegistry.registerInstance(
@@ -295,6 +313,10 @@ export function resetDependencyInjectionsForTesting() {
     ENTITY_REPOSITORY_TOKEN,
     new InMemoryTabularStorage(EntitySchema, EntityPrimaryKeyNames, [["name"], ["sic"]])
   );
+  globalServiceRegistry.registerInstance(
+    ENTITY_HISTORY_REPOSITORY_TOKEN,
+    new InMemoryTabularStorage(EntityHistorySchema, EntityHistoryPrimaryKeyNames, [["valid_to"]])
+  );
   globalServiceRegistry.registerInstance(
     ENTITY_TICKER_REPOSITORY_TOKEN,
     new InMemoryTabularStorage(EntityTickerSchema, EntityTickerPrimaryKeyNames, [