From 76168f0c8e73a797b231d9a876d9eb2f7dbfef5d Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 15 May 2026 00:07:31 -0700 Subject: [PATCH] Sync core snapshot d11f5d4 --- .env.example | 32 +++- .env.foc.local.example | 44 +++++ .env.test.example | 22 +++ .gitignore | 3 +- package-lock.json | 4 +- package.json | 2 +- src/__tests__/raw-storage-config.test.ts | 29 +++- src/config.ts | 12 ++ .../document-upload-filecoin-live.test.ts | 157 ++++++++++++++++++ 9 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 .env.foc.local.example create mode 100644 .env.test.example create mode 100644 src/services/__tests__/document-upload-filecoin-live.test.ts diff --git a/.env.example b/.env.example index 9b6ac10..db51cba 100644 --- a/.env.example +++ b/.env.example @@ -6,11 +6,11 @@ DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory # --- Provider credentials --- # Required when either EMBEDDING_PROVIDER=openai or LLM_PROVIDER=openai. -OPENAI_API_KEY=sk-... +OPENAI_API_KEY= # Required only for the matching provider selections: -# GROQ_API_KEY=gsk_... -# ANTHROPIC_API_KEY=sk-ant-... -# GOOGLE_API_KEY=... +# GROQ_API_KEY= +# ANTHROPIC_API_KEY= +# GOOGLE_API_KEY= # --- Core API auth --- # Shared API key clients must send as `Authorization: Bearer `. @@ -59,7 +59,7 @@ EMBEDDING_DIMENSIONS=1536 # Voyage embedding lane: # EMBEDDING_PROVIDER=voyage -# VOYAGE_API_KEY=pa-... +# VOYAGE_API_KEY= # VOYAGE_DOCUMENT_MODEL=voyage-4-large # VOYAGE_QUERY_MODEL=voyage-4-lite # EMBEDDING_DIMENSIONS=1024 @@ -81,13 +81,13 @@ EMBEDDING_DIMENSIONS=1536 # Other hosted LLM providers: # LLM_PROVIDER=groq -# GROQ_API_KEY=gsk_... +# GROQ_API_KEY= # LLM_MODEL=llama-3.1-8b-instant # LLM_PROVIDER=anthropic -# ANTHROPIC_API_KEY=sk-ant-... +# ANTHROPIC_API_KEY= # LLM_MODEL=claude-3-5-haiku-latest # LLM_PROVIDER=google-genai -# GOOGLE_API_KEY=... +# GOOGLE_API_KEY= # LLM_MODEL=gemini-2.0-flash # Personal local Claude Code extraction, no separate Anthropic API key: @@ -128,6 +128,22 @@ EMBEDDING_DIMENSIONS=1536 # RAW_STORAGE_LOCAL_FS_ROOT=./data/raw-storage # RAW_CONTENT_CODEC=none +# Filecoin managed blob storage: +# Filecoin data is publicly retrievable by CID unless encrypted before +# upload. Staging and production Filecoin deployments require AES-GCM. +# RAW_STORAGE_PROVIDER=filecoin +# RAW_STORAGE_DEPLOYMENT_ENV=production +# RAW_CONTENT_CODEC=aes_gcm +# RAW_CONTENT_CODEC_KEYS=v1: +# RAW_CONTENT_CODEC_ACTIVE_KEY_ID=v1 +# RAW_STORAGE_FILECOIN_DRIVER=synapse +# RAW_STORAGE_FILECOIN_NETWORK=calibration +# RAW_STORAGE_FILECOIN_PRIVATE_KEY=0x<64-hex-private-key> +# RAW_STORAGE_FILECOIN_SOURCE=atomicmemory-core +# RAW_STORAGE_FILECOIN_WITH_CDN=false +# For calibration live-test credentials, copy .env.foc.local.example to +# .env.foc.local and fill in the placeholders. Do not commit .env.foc.local. + # S3-compatible managed blob storage: # RAW_STORAGE_PROVIDER=s3 # RAW_STORAGE_S3_BUCKET=atomicmemory-raw diff --git a/.env.foc.local.example b/.env.foc.local.example new file mode 100644 index 0000000..984d830 --- /dev/null +++ b/.env.foc.local.example @@ -0,0 +1,44 @@ +# Filecoin calibration live-test environment. +# Copy to .env.foc.local and replace every placeholder before running live tests. +# Do not commit .env.foc.local; it contains a real private key. + +# Test database and local API auth used by Vitest. Start Postgres with: +# docker compose up postgres -d +DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory +CORE_API_KEY=test-core-api-key +STORAGE_KEY_HMAC_SECRET=<64-hex-character-test-secret> + +# Managed raw-storage mode under test. +RAW_STORAGE_MODE=managed_blob +RAW_STORAGE_PROVIDER=filecoin +RAW_STORAGE_PREFIX=live-filecoin +RAW_STORAGE_DEPLOYMENT_ENV=staging + +# Required for encrypted Filecoin document uploads. +RAW_CONTENT_CODEC=aes_gcm +RAW_CONTENT_CODEC_ACTIVE_KEY_ID=v1 +RAW_CONTENT_CODEC_KEYS=v1: + +# Current Filecoin provider env names consumed by core. +RAW_STORAGE_FILECOIN_DRIVER=synapse +RAW_STORAGE_FILECOIN_NETWORK=calibration +RAW_STORAGE_FILECOIN_PRIVATE_KEY=0x<64-hex-private-key> +RAW_STORAGE_FILECOIN_SOURCE=atomicmemory-core +RAW_STORAGE_FILECOIN_WITH_CDN=false + +# Legacy local aliases retained for older one-off scripts. Keep these +# aligned with the RAW_STORAGE_FILECOIN_* values above while any local +# scripts still reference RAW_STORAGE_FOC_*. +RAW_STORAGE_FOC_NETWORK=calibration +RAW_STORAGE_FOC_SOURCE=atomicmemory-core +RAW_STORAGE_FOC_PRIVATE_KEY=0x<64-hex-private-key> +RAW_STORAGE_FOC_WALLET_ADDRESS=0x<40-hex-wallet-address> + +# Optional reconciler tuning for local calibration experiments. +RAW_STORAGE_FILECOIN_RECONCILE_ENABLED=false +RAW_STORAGE_FILECOIN_RECONCILE_VERIFY=false +RAW_STORAGE_FILECOIN_RECONCILE_INTERVAL_MS=60000 +RAW_STORAGE_FILECOIN_RECONCILE_BATCH_SIZE=10 +RAW_STORAGE_FILECOIN_RECONCILE_MAX_ATTEMPTS=3 +RAW_STORAGE_FILECOIN_RECONCILE_BACKOFF_MAX_MS=600000 +RAW_STORAGE_FILECOIN_RECONCILE_STALE_AFTER_MS=900000 diff --git a/.env.test.example b/.env.test.example new file mode 100644 index 0000000..e871316 --- /dev/null +++ b/.env.test.example @@ -0,0 +1,22 @@ +# Test environment — copy to .env.test and adjust as needed. +# Requires a Postgres instance with pgvector extension. +# Use `docker compose up postgres -d` to start one locally. + +DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory +OPENAI_API_KEY=test-placeholder +CORE_API_KEY=test-core-api-key +STORAGE_KEY_HMAC_SECRET=000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f +EMBEDDING_DIMENSIONS=1024 +PORT=3051 + +# Enable PUT /memories/config for tests/local dev. Production leaves this +# unset so the route returns 410 Gone. +# See https://docs.atomicmemory.ai/platform/consuming-core. +CORE_RUNTIME_CONFIG_MUTATION_ENABLED=true + +# Phase 1 Filecoin-provider work made this knob startup-required (no +# default). Set to `local` for contributor laptops + tests; production +# deployments set `production`. Without this, every test worker crashes +# at module load with "RAW_STORAGE_DEPLOYMENT_ENV is required". +# See docs/operations/filecoin-raw-storage.md §1.1. +RAW_STORAGE_DEPLOYMENT_ENV=local diff --git a/.gitignore b/.gitignore index 90d3cb2..575fb2a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,11 @@ dist/ node_modules/ # Environment files — NEVER commit. They contain real API keys. -# Pattern below ignores every .env.* except .env.example (the template). +# Pattern below ignores every .env.* except sanitized example templates. .env .env.* !.env.example +!.env*.example .mcp.json # IDE files diff --git a/package-lock.json b/package-lock.json index 73dedb7..6be0639 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@atomicmemory/core", - "version": "1.0.1", + "version": "1.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@atomicmemory/core", - "version": "1.0.1", + "version": "1.0.2", "license": "Apache-2.0", "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.140", diff --git a/package.json b/package.json index b146aae..9bf5026 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@atomicmemory/core", - "version": "1.0.1", + "version": "1.0.2", "description": "Open-source memory engine for AI applications — semantic retrieval, AUDN mutation, and contradiction-safe claim versioning.", "type": "module", "license": "Apache-2.0", diff --git a/src/__tests__/raw-storage-config.test.ts b/src/__tests__/raw-storage-config.test.ts index 7ff1b15..5d34be2 100644 --- a/src/__tests__/raw-storage-config.test.ts +++ b/src/__tests__/raw-storage-config.test.ts @@ -157,7 +157,7 @@ describe('validateRawStorageConfig — codec keyring', () => { }); describe('validateRawStorageConfig — filecoin', () => { - it('accepts the bare provider selection in any deployment env', () => { + it('accepts encrypted provider selection in any deployment env', () => { expect(() => validateRawStorageConfig(VALID_FILECOIN_ENCRYPTED)).not.toThrow(); expect(() => validateRawStorageConfig({ ...VALID_FILECOIN_ENCRYPTED, deploymentEnv: 'staging' }), @@ -167,6 +167,33 @@ describe('validateRawStorageConfig — filecoin', () => { ).not.toThrow(); }); + it('rejects plaintext Filecoin storage outside local development', () => { + const plaintextFilecoin = { + ...VALID_FILECOIN_ENCRYPTED, + codec: 'none' as const, + codecKeys: EMPTY_RING, + codecActiveKeyId: null, + }; + expect(() => + validateRawStorageConfig({ ...plaintextFilecoin, deploymentEnv: 'production' }), + ).toThrow(/requires RAW_CONTENT_CODEC='aes_gcm'/); + expect(() => + validateRawStorageConfig({ ...plaintextFilecoin, deploymentEnv: 'staging' }), + ).toThrow(/requires RAW_CONTENT_CODEC='aes_gcm'/); + }); + + it('allows plaintext Filecoin storage only for local development', () => { + expect(() => + validateRawStorageConfig({ + ...VALID_FILECOIN_ENCRYPTED, + codec: 'none', + codecKeys: EMPTY_RING, + codecActiveKeyId: null, + deploymentEnv: 'local', + }), + ).not.toThrow(); + }); + it('does not require codec for local_fs/s3 even in production', () => { expect(() => validateRawStorageConfig({ ...VALID_LOCAL_FS, deploymentEnv: 'production' }), diff --git a/src/config.ts b/src/config.ts index fdc6b56..265b62c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -893,6 +893,17 @@ function validateRawStoragePrefix(prefix: string): void { } } +function validateFilecoinCodecPolicy(args: RawStorageValidationInput): void { + if (args.provider !== 'filecoin') return; + if (args.codec === 'aes_gcm') return; + if (args.deploymentEnv === 'local') return; + throw new Error( + "RAW_STORAGE_PROVIDER='filecoin' requires RAW_CONTENT_CODEC='aes_gcm' " + + "when RAW_STORAGE_DEPLOYMENT_ENV is 'production' or 'staging'. " + + "Plaintext Filecoin storage is only allowed for local development.", + ); +} + export function validateRawStorageConfig(args: RawStorageValidationInput): void { validateCodecConfig(args); validateLegacyProviders(args); @@ -926,6 +937,7 @@ export function validateRawStorageConfig(args: RawStorageValidationInput): void } return; } + validateFilecoinCodecPolicy(args); // provider === 'filecoin'. Synapse-shaped env validation lives in // `parseFilecoinProviderConfig` (called from this file's config // init block below) so the provider module is the single source diff --git a/src/services/__tests__/document-upload-filecoin-live.test.ts b/src/services/__tests__/document-upload-filecoin-live.test.ts new file mode 100644 index 0000000..178c60e --- /dev/null +++ b/src/services/__tests__/document-upload-filecoin-live.test.ts @@ -0,0 +1,157 @@ +/** + * @file Opt-in live integration test for encrypted document uploads to + * Filecoin. This sits above the provider smoke tests: it runs the + * document upload pipeline with a real Filecoin `RawContentStore` and + * AES-GCM codec, then verifies the remote bytes are ciphertext and + * decode back to the original plaintext. + * + * Gates: + * - `FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1` enables the suite. + * - `RAW_STORAGE_FILECOIN_*` must point at calibration credentials. + * + * Recommended invocation: + * + * FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1 \ + * dotenv -e .env.test -e .env.foc.local -- npx vitest run \ + * "src/services/__tests__/document-upload-filecoin-live.test.ts" \ + * --reporter=verbose --testTimeout=900000 + */ + +import { createHash } from 'node:crypto'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { pool } from '../../db/pool.js'; +import { clearDocumentTables, setupTestSchema } from '../../db/__tests__/test-fixtures.js'; +import { getRawDocumentById, registerRawDocument, upsertRawSource } from '../../db/raw-document-repository.js'; +import { TEST_STORAGE_KEY_HMAC_SECRET } from '../../__tests__/helpers/storage-key-test-secret.js'; +import { AesGcmRawContentCodec } from '../../storage/codecs/aes-gcm-codec.js'; +import type { InternalRawContentCodecMetadata } from '../../storage/raw-content-codec.js'; +import type { RawContentHints, RawContentStore } from '../../storage/raw-content-store.js'; +import type { FilecoinProviderConfig } from '../../storage/providers/filecoin/config.js'; +import { uploadRawDocument } from '../document-upload.js'; + +const LIVE = process.env['FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS'] === '1'; +const USER = 'filecoin-encrypted-upload-live'; +const CFG = { + rawStoragePrefix: 'live-filecoin-doc-upload', + rawStorageMode: 'managed_blob' as const, + storageKeyHmacSecret: TEST_STORAGE_KEY_HMAC_SECRET, +}; +const TEST_KEY = Buffer.alloc(32, 0x13); +const TIMEOUT_MS = 900_000; +// Mirrors the current Synapse SDK `SIZE_CONSTANTS.MIN_UPLOAD_SIZE` +// without importing the heavy vendor package outside providers/filecoin. +const SYNAPSE_MIN_UPLOAD_BYTES = 127; + +let store: RawContentStore; +let config: FilecoinProviderConfig; +let uploadedUri: string | null = null; +let uploadedHints: RawContentHints | null = null; + +function sha256Hex(buf: Buffer): string { + return createHash('sha256').update(buf).digest('hex'); +} + +function assertCalibration(cfg: FilecoinProviderConfig): void { + if (cfg.network === 'calibration') return; + throw new Error( + `document-upload-filecoin-live refuses network='${cfg.network}'. ` + + 'Set RAW_STORAGE_FILECOIN_NETWORK=calibration.', + ); +} + +function minUploadPayload(cfg: FilecoinProviderConfig): Buffer { + const size = cfg.minUploadBytes ?? SYNAPSE_MIN_UPLOAD_BYTES; + return Buffer.alloc(size, 0x45); +} + +function filecoinHints(metadata: Record): RawContentHints { + const filecoin = metadata['filecoin']; + if (!filecoin || typeof filecoin !== 'object' || Array.isArray(filecoin)) { + throw new Error('raw_storage_metadata.filecoin missing after live upload'); + } + return { filecoin: filecoin as Record }; +} + +async function seedDoc(externalId: string): Promise { + const source = await upsertRawSource(pool, { + userId: USER, + sourceSite: 'live-filecoin', + provider: 'integration-test', + }); + const registration = await registerRawDocument(pool, { + userId: USER, + rawSourceId: source.id, + externalId, + }); + return registration.document.id; +} + +describe.skipIf(!LIVE)('uploadRawDocument + Filecoin + AES-GCM live integration', () => { + beforeAll(async () => { + await setupTestSchema(pool); + const [{ parseFilecoinProviderConfig }, { createFilecoinStorageBackend }] = await Promise.all([ + import('../../storage/providers/filecoin/config.js'), + import('../../storage/providers/filecoin/index.js'), + ]); + config = parseFilecoinProviderConfig(process.env); + assertCalibration(config); + store = await createFilecoinStorageBackend(config); + }, TIMEOUT_MS); + + beforeEach(async () => { + uploadedUri = null; + uploadedHints = null; + await clearDocumentTables(pool); + }); + + afterAll(async () => { + if (uploadedUri && uploadedHints) { + const result = await store.delete(uploadedUri, uploadedHints); + expect(result.semantics).toBe('tombstoned'); + } + await clearDocumentTables(pool); + await pool.end(); + }, TIMEOUT_MS); + + it('uploads ciphertext to Filecoin and decodes retrieved bytes to the original document', async () => { + const plaintext = minUploadPayload(config); + const documentId = await seedDoc('encrypted-live-doc'); + const codec = new AesGcmRawContentCodec({ + keys: [{ keyId: 'test-v1', key: TEST_KEY }], + activeKeyId: 'test-v1', + }); + const result = await uploadRawDocument(pool, store, codec, CFG, { + userId: USER, + documentId, + body: plaintext, + }); + + uploadedUri = result.storageUri; + uploadedHints = filecoinHints(result.rawStorageMetadata); + expect(result.storageProvider).toBe('filecoin'); + expect(result.storageUri).toMatch(/^filecoin:\/\/piece\/.+$/); + expect(result.contentHash).toBe(sha256Hex(plaintext)); + expect(result.rawStorageMetadata.codec).toMatchObject({ + name: 'aes_gcm', + version: 1, + key_id: 'test-v1', + }); + + const row = await getRawDocumentById(pool, USER, documentId); + expect(row?.rawStorageMetadata).toEqual(result.rawStorageMetadata); + const providerBytes = await store.get(result.storageUri); + expect(providerBytes.body.equals(plaintext)).toBe(false); + + const decoded = await codec.decode({ + body: providerBytes.body, + metadata: result.rawStorageMetadata.codec as InternalRawContentCodecMetadata, + }); + expect(decoded.body.equals(plaintext)).toBe(true); + }, TIMEOUT_MS); +}); + +describe.skipIf(LIVE)('uploadRawDocument + Filecoin + AES-GCM live integration — gated off by default', () => { + it('skips unless FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1', () => { + expect(LIVE).toBe(false); + }); +});