Sync atomicmemory-core snapshot d11f5d4

.env.example

@@ -6,11 +6,11 @@ DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory
 
 # --- Provider credentials ---
 # Required when either EMBEDDING_PROVIDER=openai or LLM_PROVIDER=openai.
-OPENAI_API_KEY=sk-...
+OPENAI_API_KEY=<openai-api-key>
 # Required only for the matching provider selections:
-# GROQ_API_KEY=gsk_...
-# ANTHROPIC_API_KEY=sk-ant-...
-# GOOGLE_API_KEY=...
+# GROQ_API_KEY=<groq-api-key>
+# ANTHROPIC_API_KEY=<anthropic-api-key>
+# GOOGLE_API_KEY=<google-api-key>
 
 # --- Core API auth ---
 # Shared API key clients must send as `Authorization: Bearer <key>`.
@@ -59,7 +59,7 @@ EMBEDDING_DIMENSIONS=1536
 
 # Voyage embedding lane:
 # EMBEDDING_PROVIDER=voyage
-# VOYAGE_API_KEY=pa-...
+# VOYAGE_API_KEY=<voyage-api-key>
 # VOYAGE_DOCUMENT_MODEL=voyage-4-large
 # VOYAGE_QUERY_MODEL=voyage-4-lite
 # EMBEDDING_DIMENSIONS=1024
@@ -81,13 +81,13 @@ EMBEDDING_DIMENSIONS=1536
 
 # Other hosted LLM providers:
 # LLM_PROVIDER=groq
-# GROQ_API_KEY=gsk_...
+# GROQ_API_KEY=<groq-api-key>
 # LLM_MODEL=llama-3.1-8b-instant
 # LLM_PROVIDER=anthropic
-# ANTHROPIC_API_KEY=sk-ant-...
+# ANTHROPIC_API_KEY=<anthropic-api-key>
 # LLM_MODEL=claude-3-5-haiku-latest
 # LLM_PROVIDER=google-genai
-# GOOGLE_API_KEY=...
+# GOOGLE_API_KEY=<google-api-key>
 # LLM_MODEL=gemini-2.0-flash
 
 # Personal local Claude Code extraction, no separate Anthropic API key:
@@ -128,6 +128,22 @@ EMBEDDING_DIMENSIONS=1536
 # RAW_STORAGE_LOCAL_FS_ROOT=./data/raw-storage
 # RAW_CONTENT_CODEC=none
 
+# Filecoin managed blob storage:
+# Filecoin data is publicly retrievable by CID unless encrypted before
+# upload. Staging and production Filecoin deployments require AES-GCM.
+# RAW_STORAGE_PROVIDER=filecoin
+# RAW_STORAGE_DEPLOYMENT_ENV=production
+# RAW_CONTENT_CODEC=aes_gcm
+# RAW_CONTENT_CODEC_KEYS=v1:<base64url-32-byte-key>
+# RAW_CONTENT_CODEC_ACTIVE_KEY_ID=v1
+# RAW_STORAGE_FILECOIN_DRIVER=synapse
+# RAW_STORAGE_FILECOIN_NETWORK=calibration
+# RAW_STORAGE_FILECOIN_PRIVATE_KEY=0x<64-hex-private-key>
+# RAW_STORAGE_FILECOIN_SOURCE=atomicmemory-core
+# RAW_STORAGE_FILECOIN_WITH_CDN=false
+# For calibration live-test credentials, copy .env.foc.local.example to
+# .env.foc.local and fill in the placeholders. Do not commit .env.foc.local.
+
 # S3-compatible managed blob storage:
 # RAW_STORAGE_PROVIDER=s3
 # RAW_STORAGE_S3_BUCKET=atomicmemory-raw

.env.foc.local.example

@@ -0,0 +1,44 @@
+# Filecoin calibration live-test environment.
+# Copy to .env.foc.local and replace every placeholder before running live tests.
+# Do not commit .env.foc.local; it contains a real private key.
+
+# Test database and local API auth used by Vitest. Start Postgres with:
+# docker compose up postgres -d
+DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory
+CORE_API_KEY=test-core-api-key
+STORAGE_KEY_HMAC_SECRET=<64-hex-character-test-secret>
+
+# Managed raw-storage mode under test.
+RAW_STORAGE_MODE=managed_blob
+RAW_STORAGE_PROVIDER=filecoin
+RAW_STORAGE_PREFIX=live-filecoin
+RAW_STORAGE_DEPLOYMENT_ENV=staging
+
+# Required for encrypted Filecoin document uploads.
+RAW_CONTENT_CODEC=aes_gcm
+RAW_CONTENT_CODEC_ACTIVE_KEY_ID=v1
+RAW_CONTENT_CODEC_KEYS=v1:<base64url-32-byte-key>
+
+# Current Filecoin provider env names consumed by core.
+RAW_STORAGE_FILECOIN_DRIVER=synapse
+RAW_STORAGE_FILECOIN_NETWORK=calibration
+RAW_STORAGE_FILECOIN_PRIVATE_KEY=0x<64-hex-private-key>
+RAW_STORAGE_FILECOIN_SOURCE=atomicmemory-core
+RAW_STORAGE_FILECOIN_WITH_CDN=false
+
+# Legacy local aliases retained for older one-off scripts. Keep these
+# aligned with the RAW_STORAGE_FILECOIN_* values above while any local
+# scripts still reference RAW_STORAGE_FOC_*.
+RAW_STORAGE_FOC_NETWORK=calibration
+RAW_STORAGE_FOC_SOURCE=atomicmemory-core
+RAW_STORAGE_FOC_PRIVATE_KEY=0x<64-hex-private-key>
+RAW_STORAGE_FOC_WALLET_ADDRESS=0x<40-hex-wallet-address>
+
+# Optional reconciler tuning for local calibration experiments.
+RAW_STORAGE_FILECOIN_RECONCILE_ENABLED=false
+RAW_STORAGE_FILECOIN_RECONCILE_VERIFY=false
+RAW_STORAGE_FILECOIN_RECONCILE_INTERVAL_MS=60000
+RAW_STORAGE_FILECOIN_RECONCILE_BATCH_SIZE=10
+RAW_STORAGE_FILECOIN_RECONCILE_MAX_ATTEMPTS=3
+RAW_STORAGE_FILECOIN_RECONCILE_BACKOFF_MAX_MS=600000
+RAW_STORAGE_FILECOIN_RECONCILE_STALE_AFTER_MS=900000

.env.test.example

@@ -0,0 +1,22 @@
+# Test environment — copy to .env.test and adjust as needed.
+# Requires a Postgres instance with pgvector extension.
+# Use `docker compose up postgres -d` to start one locally.
+
+DATABASE_URL=postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory
+OPENAI_API_KEY=test-placeholder
+CORE_API_KEY=test-core-api-key
+STORAGE_KEY_HMAC_SECRET=000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
+EMBEDDING_DIMENSIONS=1024
+PORT=3051
+
+# Enable PUT /memories/config for tests/local dev. Production leaves this
+# unset so the route returns 410 Gone.
+# See https://docs.atomicmemory.ai/platform/consuming-core.
+CORE_RUNTIME_CONFIG_MUTATION_ENABLED=true
+
+# Phase 1 Filecoin-provider work made this knob startup-required (no
+# default). Set to `local` for contributor laptops + tests; production
+# deployments set `production`. Without this, every test worker crashes
+# at module load with "RAW_STORAGE_DEPLOYMENT_ENV is required".
+# See docs/operations/filecoin-raw-storage.md §1.1.
+RAW_STORAGE_DEPLOYMENT_ENV=local

.gitignore

@@ -6,10 +6,11 @@ dist/
 node_modules/
 
 # Environment files — NEVER commit. They contain real API keys.
-# Pattern below ignores every .env.* except .env.example (the template).
+# Pattern below ignores every .env.* except sanitized example templates.
 .env
 .env.*
 !.env.example
+!.env*.example
 .mcp.json
 
 # IDE files

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atomicmemory/core",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Open-source memory engine for AI applications — semantic retrieval, AUDN mutation, and contradiction-safe claim versioning.",
   "type": "module",
   "license": "Apache-2.0",

src/__tests__/raw-storage-config.test.ts

@@ -157,7 +157,7 @@ describe('validateRawStorageConfig — codec keyring', () => {
 });
 
 describe('validateRawStorageConfig — filecoin', () => {
-  it('accepts the bare provider selection in any deployment env', () => {
+  it('accepts encrypted provider selection in any deployment env', () => {
     expect(() => validateRawStorageConfig(VALID_FILECOIN_ENCRYPTED)).not.toThrow();
     expect(() =>
       validateRawStorageConfig({ ...VALID_FILECOIN_ENCRYPTED, deploymentEnv: 'staging' }),
@@ -167,6 +167,33 @@ describe('validateRawStorageConfig — filecoin', () => {
     ).not.toThrow();
   });
 
+  it('rejects plaintext Filecoin storage outside local development', () => {
+    const plaintextFilecoin = {
+      ...VALID_FILECOIN_ENCRYPTED,
+      codec: 'none' as const,
+      codecKeys: EMPTY_RING,
+      codecActiveKeyId: null,
+    };
+    expect(() =>
+      validateRawStorageConfig({ ...plaintextFilecoin, deploymentEnv: 'production' }),
+    ).toThrow(/requires RAW_CONTENT_CODEC='aes_gcm'/);
+    expect(() =>
+      validateRawStorageConfig({ ...plaintextFilecoin, deploymentEnv: 'staging' }),
+    ).toThrow(/requires RAW_CONTENT_CODEC='aes_gcm'/);
+  });
+
+  it('allows plaintext Filecoin storage only for local development', () => {
+    expect(() =>
+      validateRawStorageConfig({
+        ...VALID_FILECOIN_ENCRYPTED,
+        codec: 'none',
+        codecKeys: EMPTY_RING,
+        codecActiveKeyId: null,
+        deploymentEnv: 'local',
+      }),
+    ).not.toThrow();
+  });
+
   it('does not require codec for local_fs/s3 even in production', () => {
     expect(() =>
       validateRawStorageConfig({ ...VALID_LOCAL_FS, deploymentEnv: 'production' }),

src/config.ts

@@ -893,6 +893,17 @@ function validateRawStoragePrefix(prefix: string): void {
   }
 }
 
+function validateFilecoinCodecPolicy(args: RawStorageValidationInput): void {
+  if (args.provider !== 'filecoin') return;
+  if (args.codec === 'aes_gcm') return;
+  if (args.deploymentEnv === 'local') return;
+  throw new Error(
+    "RAW_STORAGE_PROVIDER='filecoin' requires RAW_CONTENT_CODEC='aes_gcm' " +
+      "when RAW_STORAGE_DEPLOYMENT_ENV is 'production' or 'staging'. " +
+      "Plaintext Filecoin storage is only allowed for local development.",
+  );
+}
+
 export function validateRawStorageConfig(args: RawStorageValidationInput): void {
   validateCodecConfig(args);
   validateLegacyProviders(args);
@@ -926,6 +937,7 @@ export function validateRawStorageConfig(args: RawStorageValidationInput): void
     }
     return;
   }
+  validateFilecoinCodecPolicy(args);
   // provider === 'filecoin'. Synapse-shaped env validation lives in
   // `parseFilecoinProviderConfig` (called from this file's config
   // init block below) so the provider module is the single source

src/services/__tests__/document-upload-filecoin-live.test.ts

@@ -0,0 +1,157 @@
+/**
+ * @file Opt-in live integration test for encrypted document uploads to
+ * Filecoin. This sits above the provider smoke tests: it runs the
+ * document upload pipeline with a real Filecoin `RawContentStore` and
+ * AES-GCM codec, then verifies the remote bytes are ciphertext and
+ * decode back to the original plaintext.
+ *
+ * Gates:
+ *   - `FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1` enables the suite.
+ *   - `RAW_STORAGE_FILECOIN_*` must point at calibration credentials.
+ *
+ * Recommended invocation:
+ *
+ *   FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1 \
+ *     dotenv -e .env.test -e .env.foc.local -- npx vitest run \
+ *     "src/services/__tests__/document-upload-filecoin-live.test.ts" \
+ *     --reporter=verbose --testTimeout=900000
+ */
+
+import { createHash } from 'node:crypto';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { pool } from '../../db/pool.js';
+import { clearDocumentTables, setupTestSchema } from '../../db/__tests__/test-fixtures.js';
+import { getRawDocumentById, registerRawDocument, upsertRawSource } from '../../db/raw-document-repository.js';
+import { TEST_STORAGE_KEY_HMAC_SECRET } from '../../__tests__/helpers/storage-key-test-secret.js';
+import { AesGcmRawContentCodec } from '../../storage/codecs/aes-gcm-codec.js';
+import type { InternalRawContentCodecMetadata } from '../../storage/raw-content-codec.js';
+import type { RawContentHints, RawContentStore } from '../../storage/raw-content-store.js';
+import type { FilecoinProviderConfig } from '../../storage/providers/filecoin/config.js';
+import { uploadRawDocument } from '../document-upload.js';
+
+const LIVE = process.env['FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS'] === '1';
+const USER = 'filecoin-encrypted-upload-live';
+const CFG = {
+  rawStoragePrefix: 'live-filecoin-doc-upload',
+  rawStorageMode: 'managed_blob' as const,
+  storageKeyHmacSecret: TEST_STORAGE_KEY_HMAC_SECRET,
+};
+const TEST_KEY = Buffer.alloc(32, 0x13);
+const TIMEOUT_MS = 900_000;
+// Mirrors the current Synapse SDK `SIZE_CONSTANTS.MIN_UPLOAD_SIZE`
+// without importing the heavy vendor package outside providers/filecoin.
+const SYNAPSE_MIN_UPLOAD_BYTES = 127;
+
+let store: RawContentStore;
+let config: FilecoinProviderConfig;
+let uploadedUri: string | null = null;
+let uploadedHints: RawContentHints | null = null;
+
+function sha256Hex(buf: Buffer): string {
+  return createHash('sha256').update(buf).digest('hex');
+}
+
+function assertCalibration(cfg: FilecoinProviderConfig): void {
+  if (cfg.network === 'calibration') return;
+  throw new Error(
+    `document-upload-filecoin-live refuses network='${cfg.network}'. ` +
+      'Set RAW_STORAGE_FILECOIN_NETWORK=calibration.',
+  );
+}
+
+function minUploadPayload(cfg: FilecoinProviderConfig): Buffer {
+  const size = cfg.minUploadBytes ?? SYNAPSE_MIN_UPLOAD_BYTES;
+  return Buffer.alloc(size, 0x45);
+}
+
+function filecoinHints(metadata: Record<string, unknown>): RawContentHints {
+  const filecoin = metadata['filecoin'];
+  if (!filecoin || typeof filecoin !== 'object' || Array.isArray(filecoin)) {
+    throw new Error('raw_storage_metadata.filecoin missing after live upload');
+  }
+  return { filecoin: filecoin as Record<string, unknown> };
+}
+
+async function seedDoc(externalId: string): Promise<string> {
+  const source = await upsertRawSource(pool, {
+    userId: USER,
+    sourceSite: 'live-filecoin',
+    provider: 'integration-test',
+  });
+  const registration = await registerRawDocument(pool, {
+    userId: USER,
+    rawSourceId: source.id,
+    externalId,
+  });
+  return registration.document.id;
+}
+
+describe.skipIf(!LIVE)('uploadRawDocument + Filecoin + AES-GCM live integration', () => {
+  beforeAll(async () => {
+    await setupTestSchema(pool);
+    const [{ parseFilecoinProviderConfig }, { createFilecoinStorageBackend }] = await Promise.all([
+      import('../../storage/providers/filecoin/config.js'),
+      import('../../storage/providers/filecoin/index.js'),
+    ]);
+    config = parseFilecoinProviderConfig(process.env);
+    assertCalibration(config);
+    store = await createFilecoinStorageBackend(config);
+  }, TIMEOUT_MS);
+
+  beforeEach(async () => {
+    uploadedUri = null;
+    uploadedHints = null;
+    await clearDocumentTables(pool);
+  });
+
+  afterAll(async () => {
+    if (uploadedUri && uploadedHints) {
+      const result = await store.delete(uploadedUri, uploadedHints);
+      expect(result.semantics).toBe('tombstoned');
+    }
+    await clearDocumentTables(pool);
+    await pool.end();
+  }, TIMEOUT_MS);
+
+  it('uploads ciphertext to Filecoin and decodes retrieved bytes to the original document', async () => {
+    const plaintext = minUploadPayload(config);
+    const documentId = await seedDoc('encrypted-live-doc');
+    const codec = new AesGcmRawContentCodec({
+      keys: [{ keyId: 'test-v1', key: TEST_KEY }],
+      activeKeyId: 'test-v1',
+    });
+    const result = await uploadRawDocument(pool, store, codec, CFG, {
+      userId: USER,
+      documentId,
+      body: plaintext,
+    });
+
+    uploadedUri = result.storageUri;
+    uploadedHints = filecoinHints(result.rawStorageMetadata);
+    expect(result.storageProvider).toBe('filecoin');
+    expect(result.storageUri).toMatch(/^filecoin:\/\/piece\/.+$/);
+    expect(result.contentHash).toBe(sha256Hex(plaintext));
+    expect(result.rawStorageMetadata.codec).toMatchObject({
+      name: 'aes_gcm',
+      version: 1,
+      key_id: 'test-v1',
+    });
+
+    const row = await getRawDocumentById(pool, USER, documentId);
+    expect(row?.rawStorageMetadata).toEqual(result.rawStorageMetadata);
+    const providerBytes = await store.get(result.storageUri);
+    expect(providerBytes.body.equals(plaintext)).toBe(false);
+
+    const decoded = await codec.decode({
+      body: providerBytes.body,
+      metadata: result.rawStorageMetadata.codec as InternalRawContentCodecMetadata,
+    });
+    expect(decoded.body.equals(plaintext)).toBe(true);
+  }, TIMEOUT_MS);
+});
+
+describe.skipIf(LIVE)('uploadRawDocument + Filecoin + AES-GCM live integration — gated off by default', () => {
+  it('skips unless FILECOIN_LIVE_DOCUMENT_UPLOAD_TESTS=1', () => {
+    expect(LIVE).toBe(false);
+  });
+});