refactor: migrate to tree-indexed sync for all categories

- Removed support for blob-based categories, transitioning to a unified tree-indexed sync approach.
- Updated category data structures and types to reflect the new sync model.
- Simplified merge and pull operations to handle only per-item categories.
- Enhanced tombstone management by separating tombstones into a dedicated file.
- Updated schema version to 5.0 to indicate the new sync structure.
- Refactored related functions and types across the codebase to align with the new architecture.

Author: ErcinDedeoglu

src/data/category-loader.ts

@@ -2,18 +2,16 @@
  * Category Loader
  *
  * Load data for sync categories.
- * Supports both blob-based (config, state, etc.) and per-item (sessions, messages) loading.
+ * All categories use per-item (tree-indexed) loading - each file synced individually.
  */
 
 import { readFile, readdir, stat } from 'node:fs/promises';
 import { join, basename } from 'node:path';
 import type { SyncCategory, PathConfig, SyncConfig } from '../types/index.js';
 import type { LocalSyncState } from '../types/sync.js';
 import type { Tombstone } from '../types/manifest.js';
-import { shouldUseItemSync } from '../types/manifest.js';
 import { getCategoryPaths } from '../types/paths.js';
-import type { CategoryData, BlobCategoryData, ItemCategoryData } from '../sync/operations/types.js';
-import { loadSinglePath, getPathKey } from './directory-loader.js';
+import type { CategoryData, ItemCategoryData } from '../sync/operations/types.js';
 import { calculateChecksum } from '../sync/item-packer.js';
 import { createTombstone, detectLocalDeletions } from '../sync/tombstone.js';
 
@@ -53,7 +51,7 @@ export async function loadLocalData(
     if (!enabledCategories[category as SyncCategory]) continue;
 
     try {
-      const data = await loadCategoryData(category as SyncCategory, paths, options);
+      const data = await loadItemCategoryData(category as SyncCategory, paths, options);
       if (data) categories.push(data);
     } catch (error) {
       const firstPath = paths[0];
@@ -71,46 +69,7 @@ export async function loadLocalData(
 }
 
 /**
- * Load data for a single category.
- * Routes to blob or per-item loading based on category.
- */
-async function loadCategoryData(
-  category: SyncCategory,
-  paths: string[],
-  options: LoadOptions
-): Promise<CategoryData | null> {
-  if (shouldUseItemSync(category)) {
-    return loadItemCategoryData(category, paths, options);
-  }
-  return loadBlobCategoryData(category, paths);
-}
-
-/**
- * Load blob-based category data (legacy approach).
- */
-async function loadBlobCategoryData(
-  category: SyncCategory,
-  paths: string[]
-): Promise<BlobCategoryData | null> {
-  const categoryData: Record<string, unknown> = {};
-  let hasData = false;
-
-  for (const basePath of paths) {
-    const result = await loadSinglePath(basePath);
-    if (result !== null) {
-      categoryData[getPathKey(basePath)] = result;
-      hasData = true;
-    }
-  }
-
-  if (!hasData) return null;
-
-  const isJsonl = category === 'state';
-  return { category, type: 'blob', data: JSON.stringify(categoryData), isJsonl };
-}
-
-/**
- * Load per-item category data (sessions, messages).
+ * Load per-item category data.
  * Each file is loaded as a separate item with its own checksum.
  * Also detects locally deleted items and creates tombstones for them.
  */

src/data/writer.ts

@@ -2,20 +2,18 @@
  * Data Writer
  *
  * Writes synced data back to local filesystem.
- * Supports both blob-based overwrite and per-item merge writes.
+ * All categories use per-item writes.
  */
 
 import { mkdir, writeFile, unlink } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import type { PathConfig, SyncCategory } from '../types/index.js';
 import { getCategoryPaths } from '../types/paths.js';
 import type { CategoryData, ItemCategoryData } from '../sync/operations/types.js';
-import { isBlobCategoryData, isItemCategoryData } from '../sync/operations/types.js';
 
 /**
  * Write synced data back to local filesystem.
- * - Blob categories: overwrites local data
- * - Item categories: merges new items (does NOT overwrite existing)
+ * All categories use per-item writes.
  */
 export async function writeLocalData(
   pathConfig: PathConfig,
@@ -27,28 +25,8 @@ export async function writeLocalData(
     const paths = categoryPaths[catData.category];
     if (paths.length === 0) continue;
 
-    if (isItemCategoryData(catData)) {
-      // Per-item merge write
-      await writeItemCategoryData(catData.category, paths, catData);
-    } else if (isBlobCategoryData(catData)) {
-      // Blob overwrite
-      const parsed = JSON.parse(catData.data) as Record<string, unknown>;
-      await writeBlobCategoryData(paths, parsed);
-    }
-  }
-}
-
-/**
- * Write blob-based category data to filesystem (overwrites).
- */
-async function writeBlobCategoryData(
-  paths: string[],
-  data: Record<string, unknown>
-): Promise<void> {
-  for (const [key, value] of Object.entries(data)) {
-    const targetPath = findTargetPath(paths, key);
-    if (targetPath === undefined) continue;
-    await writeEntry(targetPath, value);
+    // Per-item write
+    await writeItemCategoryData(catData.category, paths, catData);
   }
 }
 
@@ -99,67 +77,6 @@ async function writeItemFile(filePath: string, content: string): Promise<void> {
   }
 }
 
-/**
- * Find target path for a key.
- */
-function findTargetPath(paths: string[], key: string): string | undefined {
-  return paths.find((p) => p.endsWith(key)) ?? paths[0];
-}
-
-/**
- * Write a single entry (file or directory).
- */
-async function writeEntry(targetPath: string, value: unknown): Promise<void> {
-  try {
-    if (isDirectoryValue(value)) {
-      await mkdir(targetPath, { recursive: true });
-      await writeDirectoryData(targetPath, value as Record<string, unknown>);
-    } else {
-      await ensureParentDir(targetPath);
-      const content = serializeContent(targetPath, value);
-      await writeFile(targetPath, content, 'utf-8');
-    }
-  } catch (error) {
-    console.error(`Failed to write ${targetPath}:`, error);
-  }
-}
-
-/**
- * Check if value represents a directory.
- */
-function isDirectoryValue(value: unknown): boolean {
-  return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-
-/**
- * Write directory contents recursively.
- */
-async function writeDirectoryData(dirPath: string, data: Record<string, unknown>): Promise<void> {
-  for (const [name, value] of Object.entries(data)) {
-    const fullPath = join(dirPath, name);
-    await writeEntry(fullPath, value);
-  }
-}
-
-/**
- * Serialize content based on file type.
- */
-function serializeContent(filePath: string, value: unknown): string {
-  if (filePath.endsWith('.json')) {
-    return JSON.stringify(value, null, 2);
-  }
-
-  if (filePath.endsWith('.jsonl') && Array.isArray(value)) {
-    return value.map((item) => JSON.stringify(item)).join('\n');
-  }
-
-  if (typeof value === 'string') {
-    return value;
-  }
-
-  return JSON.stringify(value, null, 2);
-}
-
 /**
  * Ensure parent directory exists.
  */

src/sync/engine/helpers.ts

@@ -12,13 +12,13 @@ import type {
   Tombstone,
 } from '../../types/index.js';
 import type { CategoryData, PassphraseOption } from '../operations/types.js';
-import { isItemCategoryData } from '../operations/types.js';
 import type { PullOptions } from '../operations/pull.js';
 import type { PreparePushResult } from '../operations/push.js';
 import { preparePushData } from '../operations/push.js';
 
 /**
- * Build checksums map from local item category data for merge-based pull.
+ * Build checksums map from local category data for merge-based pull.
+ * All categories use per-item sync.
  */
 export function buildLocalChecksums(
   localData?: CategoryData[]
@@ -31,9 +31,7 @@ export function buildLocalChecksums(
   >;
 
   for (const catData of localData) {
-    if (isItemCategoryData(catData)) {
-      checksums[catData.category] = catData.checksums;
-    }
+    checksums[catData.category] = catData.checksums;
   }
 
   return Object.keys(checksums).length > 0 ? checksums : undefined;

src/sync/engine/index.ts

@@ -9,7 +9,7 @@ export { SyncError } from './errors.js';
 export type { SyncEngineOptions } from './types.js';
 export { MANIFEST_FILENAME } from './types.js';
 export { fetchManifest } from './manifest.js';
-export { buildLocalState, isLockedByOther, getStorageFilesMap } from './state.js';
+export { buildLocalState, isLockedByOther } from './state.js';
 export {
   buildPushResult,
   buildPullResult,

src/sync/engine/operations.ts

@@ -11,7 +11,7 @@ import { pullCategories } from '../operations/pull.js';
 import { mergeAllCategories } from '../operations/merge-operation.js';
 import { syncLog } from './logger.js';
 import { MANIFEST_FILENAME } from './types.js';
-import { buildLocalState, getStorageFilesMap, mergeDataForState } from './state.js';
+import { buildLocalState, mergeDataForState } from './state.js';
 import {
   buildPushResult,
   buildPullResult,
@@ -72,11 +72,9 @@ export async function executePullOperation(
   data?: CategoryData[]
 ): Promise<{ result: SyncResult; newState: LocalSyncState | null }> {
   if (!remote) return { result: buildErrorResult('No remote data found'), newState: null };
-  const sf = await getStorageFilesMap(ctx.backend);
   const opts = buildPullOptions(
     {
       manifest: remote,
-      storageFiles: sf,
       enabledCategories: ctx.config.sync,
       passphrase: buildCryptoOptions(ctx.passphrase, ctx.oldPassphrase),
       backend: ctx.backend,
@@ -105,15 +103,13 @@ export async function executeConflictOperation(
   manifest: Manifest,
   retry: number
 ): Promise<SyncResult> {
-  const sf = await getStorageFilesMap(ctx.backend);
   const mergeCtx = {
     remoteManifest: manifest,
-    storageFiles: sf,
     localState: ctx.localState,
     passphrase: buildCryptoOptions(ctx.passphrase, ctx.oldPassphrase),
     machineId: ctx.config.machineId,
     backend: ctx.backend,
   };
-  const { mergedData, conflicts } = await mergeAllCategories(data, mergeCtx);
+  const { mergedData, conflicts } = mergeAllCategories(data, mergeCtx);
   return buildConflictResult(await ctx.pushFn(mergedData, retry, manifest), conflicts);
 }

src/sync/engine/routing.ts

@@ -31,16 +31,15 @@ export function determineAction(
   const remoteTimestamp = remoteManifest.updatedAt;
 
   const cmp = compareTimestamps(localTimestamp, remoteTimestamp);
-  const pushNeeded = cmp === 'equal' ? needsPush(localData, remoteManifest) : false;
+  // Check needsPush for both 'equal' and 'local-newer' cases
+  const pushNeeded = cmp !== 'remote-newer' ? needsPush(localData, remoteManifest) : false;
   syncLog(`[SYNC] Timestamp: ${cmp}, needsPush=${String(pushNeeded)}`);
 
   switch (cmp) {
     case 'equal':
-      // Same timestamp - use checksum to decide
-      return pushNeeded ? { action: 'push' } : { action: 'no-change' };
     case 'local-newer':
-      // Local has newer changes - push them
-      return { action: 'push' };
+      // Same or local newer - only push if there are actual changes
+      return pushNeeded ? { action: 'push' } : { action: 'no-change' };
     case 'remote-newer':
       // Remote has newer changes - pull first, then push if local has changes
       // This implements last-write-wins: remote wins, we'll push our changes after

src/sync/engine/state.ts

@@ -4,15 +4,12 @@
  * Handles local state updates after sync operations.
  */
 
-import { calculateChecksum } from '../packer.js';
-import type { StorageBackend } from '../../storage/index.js';
 import type { Manifest, LocalSyncState, SyncCategory } from '../../types/index.js';
-import type { CategoryData, StorageFiles } from '../operations/types.js';
-import { isBlobCategoryData, isItemCategoryData } from '../operations/types.js';
+import type { CategoryData } from '../operations/types.js';
 
 /**
  * Build updated local state after a sync operation.
- * Only tracks base versions for blob categories (used for three-way merge).
+ * All categories use per-item tracking.
  */
 export function buildLocalState(
   manifest: Manifest,
@@ -26,14 +23,8 @@ export function buildLocalState(
   const itemChecksums: Partial<Record<SyncCategory, Record<string, string>>> = {};
 
   for (const item of data) {
-    if (isBlobCategoryData(item)) {
-      // Blob categories: track single checksum and base version for three-way merge
-      checksums[item.category] = calculateChecksum(item.data);
-      baseVersions[item.category] = item.data;
-    } else if (isItemCategoryData(item)) {
-      // Per-item categories: track individual item checksums for deletion detection
-      itemChecksums[item.category] = item.checksums;
-    }
+    // All categories: track individual item checksums for deletion detection
+    itemChecksums[item.category] = item.checksums;
   }
 
   return {
@@ -71,9 +62,7 @@ function buildCategoryMap(data: CategoryData[]): Map<SyncCategory, CategoryData>
 }
 
 /** Merge item category data (local + pulled) */
-import type { ItemCategoryData } from '../operations/types.js';
-
-function mergeItemData(local: ItemCategoryData, pulled: ItemCategoryData): CategoryData {
+function mergeItemData(local: CategoryData, pulled: CategoryData): CategoryData {
   return {
     category: pulled.category,
     type: 'items',
@@ -87,11 +76,8 @@ function processPulledCategory(
   pulled: CategoryData,
   localByCategory: Map<SyncCategory, CategoryData>
 ): CategoryData {
-  if (isBlobCategoryData(pulled)) return pulled;
-  if (!isItemCategoryData(pulled)) return pulled;
-
   const local = localByCategory.get(pulled.category);
-  if (local && isItemCategoryData(local)) {
+  if (local) {
     return mergeItemData(local, pulled);
   }
   return pulled;
@@ -124,18 +110,3 @@ export function mergeDataForState(
 
   return result;
 }
-
-/**
- * Build storage files map from backend listing.
- */
-export async function getStorageFilesMap(backend: StorageBackend): Promise<StorageFiles> {
-  const files = await backend.listFiles();
-  const map: StorageFiles = {};
-  for (const file of files) {
-    const entry: { content?: string; sha?: string } = {};
-    if (file.content !== undefined) entry.content = file.content;
-    if (file.sha !== undefined) entry.sha = file.sha;
-    map[file.filename] = entry;
-  }
-  return map;
-}

src/sync/index.ts

@@ -11,7 +11,7 @@ export type { SyncEngineOptions } from './engine/index.js';
 // Operations
 export type { CategoryData } from './operations/types.js';
 export { preparePushData, needsPush } from './operations/push.js';
-export { pullCategories, downloadChunks } from './operations/pull.js';
+export { pullCategories } from './operations/pull.js';
 export { mergeAllCategories } from './operations/merge-operation.js';
 
 // Timestamp-based sync (replaces vector clocks)
@@ -40,12 +40,11 @@ export {
   createTombstone,
   isTombstoneExpired,
   filterExpiredTombstones,
-  cleanupExpiredTombstones,
   mergeTombstones,
   isItemTombstoned,
   getItemsToDelete,
   detectLocalDeletions,
-  // TombstonesFile handling (schema 4.0)
+  // TombstonesFile handling (schema 5.0)
   parseTombstonesFile,
   serializeTombstonesFile,
   getCategoryTombstones,