feat(sync): enhance sync functionality with local state persistence and improved data merging

Author: ErcinDedeoglu

docs/CONFIG.md

@@ -99,6 +99,10 @@ On subsequent runs:
 | `fileWatcherDebounceMs` | number | `5000` | Wait after file change before sync |
 | `maxDebounceMs` | number | `30000` | Max wait during heavy activity |
 | `conflictStrategy` | string | `auto-merge` | How to resolve conflicts |
+| `advisoryLockTimeoutSeconds` | number | `30` | Lock timeout for concurrent sync prevention |
+| `maxRetryAttempts` | number | `3` | Max retries on transient failures |
+| `retryDelayMs` | number | `1000` | Delay between retry attempts |
+| `tombstoneGraceDays` | number | `30` | Days before deleted item markers expire |
 
 ## Sync Categories
 
@@ -151,6 +155,12 @@ All categories are enabled by default. Disable `messages` if you have very large
     "projects": true,
     "todos": true
   },
-  "conflictStrategy": "auto-merge"
+  "conflictStrategy": "auto-merge",
+  "advisoryLockTimeoutSeconds": 30,
+  "maxRetryAttempts": 3,
+  "retryDelayMs": 1000,
+  "tombstoneGraceDays": 30
 }
 ```
+
+Note: `machineId`, `keySalt`, and `passphraseHash` are auto-generated and should not be manually edited.

src/plugin/plugin.ts

@@ -14,7 +14,7 @@ import {
   updateConfig,
   initializeEngine,
 } from './state-manager.js';
-import { getTokenSource, loadLocalData } from '../data/index.js';
+import { getTokenSource, loadLocalData, saveLocalState } from '../data/index.js';
 import { RepoStorageBackend } from '../storage/index.js';
 import { FileWatcher } from '../sync/watcher/index.js';
 import type { PluginState } from './types.js';
@@ -187,6 +187,16 @@ async function ensureStorageExists(pathConfig: PathConfig): Promise<void> {
   }
 }
 
+/** Persist engine's local state to disk after successful sync */
+async function persistLocalState(pathConfig: PathConfig): Promise<void> {
+  const state = getPluginState();
+  const newState = state.engine?.getLocalState();
+  if (newState) {
+    await saveLocalState(pathConfig, newState);
+    state.localState = newState;
+  }
+}
+
 /** Perform initial sync on plugin startup (non-blocking) */
 function performInitialSync(pathConfig: PathConfig): void {
   const state = getPluginState();
@@ -211,6 +221,7 @@ function performInitialSync(pathConfig: PathConfig): void {
       const result = await engine.sync(categories);
 
       if (result.success && result.action !== 'error') {
+        await persistLocalState(pathConfig);
         log(`Initial sync complete: ${result.message}`);
       } else {
         log(`Sync completed: ${result.message}`);
@@ -248,7 +259,10 @@ function startFileWatcher(pathConfig: PathConfig): void {
       onEvent: async () => {
         try {
           const { categories } = await loadLocalData(pathConfig, config.sync);
-          await engine.sync(categories);
+          const result = await engine.sync(categories);
+          if (result.success) {
+            await persistLocalState(pathConfig);
+          }
         } catch (error) {
           const errMsg = error instanceof Error ? error.message : String(error);
           log(`WARNING: File watcher sync failed: ${errMsg}`);
@@ -279,7 +293,21 @@ function startIntervalSync(pathConfig: PathConfig): void {
     void (async () => {
       try {
         const { categories } = await loadLocalData(pathConfig, config.sync);
-        await engine.sync(categories);
+        const result = await engine.sync(categories);
+        // Persist state after successful sync
+        if (result.success) {
+          await persistLocalState(pathConfig);
+        }
+        // Log interval sync results (both success and no-change)
+        if (result.action === 'pushed') {
+          log(`Interval sync: ${result.message}`);
+        } else if (result.action === 'pulled') {
+          log(`Interval sync: ${result.message}`);
+        } else if (result.action === 'no-change') {
+          // Don't log no-change to keep logs clean
+        } else if (result.action === 'error') {
+          log(`WARNING: Interval sync error: ${result.message}`);
+        }
       } catch (error) {
         const errMsg = error instanceof Error ? error.message : String(error);
         log(`WARNING: Interval sync failed: ${errMsg}`);

src/sync/engine/state.ts

@@ -6,9 +6,9 @@
 
 import { calculateChecksum } from '../packer.js';
 import type { StorageBackend } from '../../storage/index.js';
-import type { Manifest, LocalSyncState } from '../../types/index.js';
+import type { Manifest, LocalSyncState, SyncCategory } from '../../types/index.js';
 import type { CategoryData, StorageFiles } from '../operations/types.js';
-import { isBlobCategoryData } from '../operations/types.js';
+import { isBlobCategoryData, isItemCategoryData } from '../operations/types.js';
 
 /**
  * Build updated local state after a sync operation.
@@ -23,14 +23,17 @@ export function buildLocalState(
   const now = new Date().toISOString();
   const checksums: LocalSyncState['categoryChecksums'] = {};
   const baseVersions: LocalSyncState['baseVersions'] = {};
+  const itemChecksums: Partial<Record<SyncCategory, Record<string, string>>> = {};
 
   for (const item of data) {
-    // Only blob categories have a single data string for checksum/base tracking
     if (isBlobCategoryData(item)) {
+      // Blob categories: track single checksum and base version for three-way merge
       checksums[item.category] = calculateChecksum(item.data);
       baseVersions[item.category] = item.data;
+    } else if (isItemCategoryData(item)) {
+      // Per-item categories: track individual item checksums for deletion detection
+      itemChecksums[item.category] = item.checksums;
     }
-    // Per-item categories don't need base versions (they use additive merge)
   }
 
   return {
@@ -41,6 +44,7 @@ export function buildLocalState(
     vectorClock: manifest.vectorClock,
     categoryChecksums: checksums,
     baseVersions,
+    itemChecksums,
   };
 }
 
@@ -60,6 +64,68 @@ export function isLockedByOther(
   return Date.now() - lockTime < timeout;
 }
 
+/** Build a lookup map from category array */
+function buildCategoryMap(data: CategoryData[]): Map<SyncCategory, CategoryData> {
+  const map = new Map<SyncCategory, CategoryData>();
+  for (const item of data) map.set(item.category, item);
+  return map;
+}
+
+/** Merge item category data (local + pulled) */
+import type { ItemCategoryData } from '../operations/types.js';
+
+function mergeItemData(local: ItemCategoryData, pulled: ItemCategoryData): CategoryData {
+  return {
+    category: pulled.category,
+    type: 'items',
+    items: { ...local.items, ...pulled.items },
+    checksums: { ...local.checksums, ...pulled.checksums },
+  };
+}
+
+/** Process a single pulled category, merging with local if needed */
+function processPulledCategory(
+  pulled: CategoryData,
+  localByCategory: Map<SyncCategory, CategoryData>
+): CategoryData {
+  if (isBlobCategoryData(pulled)) return pulled;
+  if (!isItemCategoryData(pulled)) return pulled;
+
+  const local = localByCategory.get(pulled.category);
+  if (local && isItemCategoryData(local)) {
+    return mergeItemData(local, pulled);
+  }
+  return pulled;
+}
+
+/**
+ * Merge local and pulled data for complete state tracking.
+ * After a pull, we need to track ALL items (local + pulled) for proper deletion detection.
+ */
+export function mergeDataForState(
+  localData: CategoryData[] | undefined,
+  pulledData: CategoryData[]
+): CategoryData[] {
+  if (!localData) return pulledData;
+
+  const localByCategory = buildCategoryMap(localData);
+  const processedCategories = new Set<SyncCategory>();
+  const result: CategoryData[] = [];
+
+  // Process pulled data first (it takes precedence for blobs)
+  for (const pulled of pulledData) {
+    processedCategories.add(pulled.category);
+    result.push(processPulledCategory(pulled, localByCategory));
+  }
+
+  // Add local-only categories (not in pulled)
+  for (const local of localData) {
+    if (!processedCategories.has(local.category)) result.push(local);
+  }
+
+  return result;
+}
+
 /**
  * Build storage files map from backend listing.
  */

src/sync/engine/sync-engine.ts

@@ -14,7 +14,12 @@ import type { CategoryData } from '../operations/types.js';
 import type { SyncEngineOptions } from './types.js';
 import { MANIFEST_FILENAME } from './types.js';
 import { fetchManifest } from './manifest.js';
-import { buildLocalState, isLockedByOther, getStorageFilesMap } from './state.js';
+import {
+  buildLocalState,
+  isLockedByOther,
+  getStorageFilesMap,
+  mergeDataForState,
+} from './state.js';
 import {
   buildPushResult,
   buildPullResult,
@@ -180,7 +185,10 @@ export class SyncEngine {
       data
     );
     const { pulledData, changedCategories, tombstonedItems } = await pullCategories(opts);
-    this.localState = buildLocalState(m, pulledData, this.getStorageId(), this.config.machineId);
+    // Merge local and pulled data for complete state tracking
+    // This ensures local-only items are tracked for subsequent push detection
+    const mergedData = mergeDataForState(data, pulledData);
+    this.localState = buildLocalState(m, mergedData, this.getStorageId(), this.config.machineId);
     // Convert tombstonedItems to item ID arrays for the result
     const tombstoneIds = extractTombstoneIds(tombstonedItems);
     return buildPullResult({ changedCategories, pulledData, tombstonedItems: tombstoneIds });