diff --git a/.gitignore b/.gitignore index 9b3d016..4381717 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules/ dist/ *.vsix site/ +.memory/cleanup.log diff --git a/.memory/cleanup.log b/.memory/cleanup.log deleted file mode 100644 index 85db423..0000000 --- a/.memory/cleanup.log +++ /dev/null @@ -1,10 +0,0 @@ - ---- Cleanup: 2026-02-27T13:49:52.191Z --- -Merged 2 similar entries → "ADRs live in docs/decisions/ but are NOT in the Zensical nav…" -Pruned [Decision] (score 22): "Model resolution always happens inside lm.ts. Call…" - ---- Cleanup: 2026-02-27T15:01:41.609Z --- -Merged 2 similar entries → "Use replaceAll() instead of replace() for string replacement…" -Pruned [Preference] (score 15): "Dedup thresholds guard writes. Cleanup merge thres…" -Pruned [Preference] (score 14): "Use replaceAll() instead of replace() for string r…" -Pruned [Decision] (score 20): "The default category limit fallback is 20, hardcod…" diff --git a/.memory/decisions.md b/.memory/decisions.md index 6a16f25..db8dc35 100644 --- a/.memory/decisions.md +++ b/.memory/decisions.md @@ -34,12 +34,10 @@ Memories stored by HackLM Memory. - [docs-site-zensical] Docs site uses Zensical (pip install zensical). Config in zensical.toml at repo root. Serve: .venv/Scripts/zensical.exe serve. Build: zensical build → site/. Deploys to GitHub Pages via .github/workflows/docs.yml on push to main. - - [globalstate-keys-file] All globalState keys live in extension/src/globalStateKeys.ts as named exports. Never use bare string literals for globalState keys elsewhere. - [category-limits-source] CATEGORY_LIMITS constant removed from markdownStore.ts. Category limits come from VS Code config only (package.json defaults via config.get). Default fallback is hardcoded as 20 in utils.ts. -- [stale-slug-pruner-removed] pruneStaleEntries removed from cleanupMemory.ts. Stale slugs last-cleanup and last-session-debrief no longer exist. ADRs 0001 and 0013 deleted from docs/decisions/ as they documented completed migration periods. - - - [adr-location] ADRs live in docs/decisions.md but are NOT in the Zensical nav. The nav has an ADR page (docs/adr.md) instead. That page links back to docs/decisions.md on GitHub. + +- [category-limits-defaults] Category defaults doubled: Instruction/Security = 30, Quirk/Preference/Decision = 40. Schema maximum raised to 100. Fallback in utils.ts and UI validation cap updated to match. diff --git a/.memory/quirks.md b/.memory/quirks.md index 2c4d167..e961ee4 100644 --- a/.memory/quirks.md +++ b/.memory/quirks.md @@ -9,3 +9,5 @@ Memories stored by HackLM Memory. - [empty-justification] Passing an empty options object to sendRequest suppresses the justification string in the VS Code permission prompt. The user sees a blank reason. Always pass a justification. - [adr-location] ADRs live in docs/decisions.md (a single flat file, not a directory). No separate per-ADR files. Add new ADRs as new H2 sections at the bottom of that file. + +- [wx-flag-advisory-lock] The outer cross-process lock uses fs.open with the wx flag to create the lockfile exclusively. If the file already exists the open fails, signaling another process holds the lock. Never use a different open mode for this file. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6935d11..72dbacb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2026-02-28 + +### Added + +- Two-tier write locking: cross-process advisory lockfile (`crossProcessLock.ts`) stacked over per-file in-process promise queue — prevents concurrent writes from multiple VS Code windows +- Plan agent patching: `patchPlanAgent()` injects `queryMemory` + `storeMemory` into the Copilot Chat Plan agent on every activation (idempotent) + +### Changed + +- Default category limits doubled: Instruction/Security 15→30, Quirk/Preference/Decision 20→40 +- Configurable maximum raised from 50 to 100 + ## [1.0.0] - 2026-02-27 ### Added @@ -30,7 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Per-category entry limit settings - Getting Started walkthrough (5 steps) - esbuild single-file bundle (`dist/extension.js`) — no webpack -- Publishes to VS Marketplace and Open VSX (compatible with VS Code and Google Antigravity) +- Publishes to VS Marketplace and Open VSX [Unreleased]: https://github.com/hacklmdev/memory/compare/v1.0.0...HEAD [1.0.0]: https://github.com/hacklmdev/memory/releases/tag/v1.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9077a01..acdff64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thank you for your interest in contributing. ## Prerequisites - [Node.js](https://nodejs.org/) 20+ -- [VS Code](https://code.visualstudio.com/) 1.99+ (or any Open VSX-compatible editor, e.g. Google Antigravity) +- [VS Code](https://code.visualstudio.com/) 1.99+ - Git ## Build diff --git a/README.md b/README.md index 97f1f27..8fe7d7c 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,27 @@ A long-term memory layer for VS Code Copilot — learn from interactions, retain ## Compatibility -Works in any Open VSX-compatible editor: +Requires [VS Code](https://code.visualstudio.com/) 1.99+. -- [VS Code](https://code.visualstudio.com/) 1.99+ -- [Google Antigravity](https://antigravity.google/) (Open VSX) -- [Gitpod](https://gitpod.io/) (Open VSX) +Also published to [Open VSX](https://open-vsx.org/extension/hacklm/hacklm-memory) for editors that use the Open VSX registry. -> **Note:** The memory LM tools (`storeMemory`, `queryMemory`) require the editor to implement the VS Code LM Tool API. Editors that do not implement this API will install the extension but the tools will not be available in chat. +### Open VSX editors + +In Open VSX-compatible editors the extension installs and activates. What works depends on whether the editor implements the VS Code LM Tool API: + +| Feature | Available | +|---------|----------| +| `.memory/*.md` files created on first open | Always | +| `copilot-instructions.md` reference block injected | Always | +| `storeMemory` / `queryMemory` LM tools | VS Code LM Tool API required | +| Tree view, status bar, control panel | VS Code only | +| LM deduplication and gap analysis | VS Code LM Tool API required | + +The `.memory/` files and `copilot-instructions.md` pointer are set up regardless of editor. Once those exist, any AI agent that reads instruction files can access the memory store directly — even without the LM tools. + +The full experience (LM tools, UI) requires VS Code 1.99+. + +Support for additional editors via MCP is planned — see the [Roadmap](docs/roadmap.md). ## Architecture @@ -78,7 +92,7 @@ npm run build | `hacklm-memory.lmFamily` | `gpt-5-mini` | Copilot model family for LM operations | | `hacklm-memory.autoApproveStore` | `false` | Skip confirmation prompt when saving memories | | `hacklm-memory.manageInstructionFile` | `true` | Allow the extension to manage `.github/copilot-instructions.md` | -| `hacklm-memory.categoryLimit.*` | varies | Per-category max entry counts | +| `hacklm-memory.categoryLimit.*` | varies (30–40) | Per-category max entry counts | ## Privacy @@ -94,7 +108,7 @@ Contributions are welcome. Please read [CONTRIBUTING.md](CONTRIBUTING.md) before - [Architecture](docs/architecture.md) — module map, data flow, key design decisions - [API Reference](docs/api-reference.md) — LM tool schemas, memory file format - [Developer Guide](docs/contributing.md) — how to add categories, tools, and tests -- [Architecture Decision Records](docs/decisions.md) — 16 ADRs covering every major decision +- [Architecture Decision Records](docs/decisions.md) — 17 ADRs covering every major decision - [Roadmap](docs/roadmap.md) — planned work and open contribution areas ## License diff --git a/docs/adr.md b/docs/adr.md index 0542b6f..a71aef7 100644 --- a/docs/adr.md +++ b/docs/adr.md @@ -32,9 +32,9 @@ This makes the codebase predictable. When LM behaviour changes, you edit one fil ## 4. Editor-agnostic storage -The `.memory/*.md` file format is the stable contract — not VS Code, not TypeScript, not this extension. A future JetBrains plugin or Neovim integration must read and write the same format for memory to be shareable across editors. +The `.memory/*.md` file format is the stable contract — not VS Code, not TypeScript, not this extension. Any future port (JetBrains plugin, Neovim integration, MCP server for Google Antigravity) must read and write the same format for memory to be shareable across editors. -The storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) has no VS Code dependency and must stay that way. +The storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) has no VS Code dependency and must stay that way. When additional editor support is built, this layer is extracted to `packages/storage` and shared — not copied. --- diff --git a/docs/api-reference.md b/docs/api-reference.md index 2d5d309..b93e340 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -116,11 +116,11 @@ Returns `"No memories found."` if no entries match. | Category | File | Default Limit | What It Stores | |----------|------|---------------|----------------| -| `Instruction` | `.memory/instructions.md` | 15 | How Copilot should behave | -| `Quirk` | `.memory/quirks.md` | 20 | Project-specific weirdness — non-obvious gotchas | -| `Preference` | `.memory/preferences.md` | 20 | Style, tone, and design choices | -| `Decision` | `.memory/decisions.md` | 20 | Architectural commitments | -| `Security` | `.memory/security.md` | 15 | Rules that must never be broken | +| `Instruction` | `.memory/instructions.md` | 30 | How Copilot should behave | +| `Quirk` | `.memory/quirks.md` | 40 | Project-specific weirdness — non-obvious gotchas | +| `Preference` | `.memory/preferences.md` | 40 | Style, tone, and design choices | +| `Decision` | `.memory/decisions.md` | 40 | Architectural commitments | +| `Security` | `.memory/security.md` | 30 | Rules that must never be broken | Limits are configurable per-category via `hacklm-memory.categoryLimit.` settings. diff --git a/docs/architecture.md b/docs/architecture.md index 8c6ab64..8f4b5fd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,9 +69,17 @@ flowchart TD ## Key Design Decisions -### Per-file Mutex Locks +### Two-tier Write Locks -`markdownStore.ts` maintains a `Map>` as a chained promise queue. Every read-then-write operation on a `.memory/*.md` file acquires the lock for that specific file. This prevents concurrent `storeMemory` calls and background cleanup from corrupting the same file simultaneously. +Every write to `.memory/*.md` goes through two stacked locks: + +1. **Outer — cross-process advisory lockfile** (`crossProcessLock.ts`): `withCrossProcessLock(memoryDir, fn)` creates `.memory/.lock` with an exclusive `fs.open('wx')` — an atomic OS primitive that works on POSIX and NTFS. Only one process can hold this file at a time. All other processes spin (with linear back-off, max 20 retries) until the holder removes it. Stale locks (dead PID or age > 10 s) are automatically removed, including on extension startup via `clearStaleLockOnStartup()`. + +2. **Inner — per-file in-process promise queue** (`markdownStore.ts`): `withFileLock(filePath, fn)` chains a `Promise` per file path. This serialises concurrent async calls within the same extension host process (e.g. `storeMemory` + background cleanup hitting the same category file). + +Callers outside `markdownStore.ts` never interact with either lock directly. The combined wrapper `withMemoryWriteLock(filePath, fn)` is the only entry point used by `appendMemory`, `upsertMemory`, `deleteMemory`, and `migrateFiles`. + +See [ADR 0018](decisions.md#0018--two-tier-write-locking-for-memory-files) for the full decision record. ### Centralised LM Access @@ -99,7 +107,7 @@ This extension uses: - `vscode.lm.selectChatModels` — to select a Copilot model for redundancy checks and gap analysis - `vscode.lm.onDidChangeChatModels` — to refresh the status bar when available models change -These APIs require **VS Code 1.99+** (or any Open VSX-compatible editor that implements the VS Code LM Tool API at the same version level). Editors that do not implement `vscode.lm.registerTool` will install the extension but the memory tools will not be available in chat. +These APIs require **VS Code 1.99+**. The extension publishes to Open VSX; the memory tools function in any editor that implements the VS Code LM Tool API. A separate MCP-based implementation for additional editors is planned (see [ADR 0017](decisions.md#0017--additional-editor-support-via-mcp)). ## Future: JetBrains Support diff --git a/docs/decisions.md b/docs/decisions.md index e0b257f..e8e7b76 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -251,11 +251,11 @@ Each memory category has a maximum entry count enforced during cleanup. Limits a | Category | Limit | Rationale | |----------|-------|-----------| -| Instruction | 15 | Low count, high value. Instructions should be crisp. | -| Decision | 20 | Grows fast on active projects. | -| Quirk | 20 | Accumulates over time. Cleanup keeps it relevant. | -| Preference | 20 | Style choices accumulate. | -| Security | 15 | Should be small and authoritative. | +| Instruction | 30 | Low count, high value. Instructions should be crisp. | +| Decision | 40 | Grows fast on active projects. | +| Quirk | 40 | Accumulates over time. Cleanup keeps it relevant. | +| Preference | 40 | Style choices accumulate. | +| Security | 30 | Should be small and authoritative. | All limits are user-configurable via `hacklm-memory.categoryLimit.` settings. @@ -315,11 +315,70 @@ HackLM Memory could theoretically support multiple editors. JetBrains requires a ### Decision -The initial release targets **VS Code-based editors only**: VS Code and any Open VSX-compatible editor. JetBrains support is deferred until a maintainer with JetBrains plugin experience volunteers. +The initial release targets **VS Code**. JetBrains support is deferred until a maintainer with JetBrains plugin experience volunteers. Support for additional editors will be delivered via MCP (see ADR 0017). ### Consequences - The extension is written in TypeScript against the VS Code extension API exclusively. -- The VS Code LM Tool API (`vscode.lm.registerTool`, `vscode.lm.selectChatModels`) has no JetBrains equivalent. -- The storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) has no `vscode` dependency and could be extracted into a `core/` package for a future JetBrains port. +- The storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) has no `vscode` dependency and could be extracted into a `packages/storage` package for future ports. - The `.memory/*.md` file format (documented in [`api-reference.md`](api-reference.md)) is the stable, editor-agnostic contract any future implementation must honour. + +--- + +## 0017 — Additional Editor Support via MCP + +**Status:** Accepted (not yet implemented) + +### Context + +Some editors (e.g. Google Antigravity) support MCP (Model Context Protocol), which allows agents to call external tools. These editors do not share the VS Code extension API surface. + +### Decision + +Support for MCP-capable editors will be delivered as a **separate MCP server** (`mcp/`) rather than by modifying the existing extension. The current extension remains VS Code-focused with no changes. + +When the MCP server is built: +- The storage layer is extracted to `packages/storage` (shared by both `extension/` and `mcp/`). +- The MCP server accepts `workspaceRoot` as a per-call parameter (no VS Code workspace context). +- LM-based redundancy checks are omitted — Jaccard fuzzy matching is sufficient without a second LM pass. +- Gap analysis and session review are omitted — no user-prompt UI exists in the MCP context. + +### Consequences + +- Zero changes to the existing VS Code extension. +- No code duplication — storage logic lives once in `packages/storage`. +- The MCP server is a clean, dependency-light Node.js process. +- Feature parity is intentionally incomplete: gap analysis and LM dedup are VS Code-only features. + +--- + +## 0018 — Two-tier Write Locking for Memory Files + +**Status:** Accepted + +### Context + +The original in-process `withFileLock` (a chained-promise mutex keyed by file path) only protects concurrent calls within a single extension host process. When multiple agents run in the same worktree — each in its own process — simultaneous read-modify-write operations on the same `.memory/*.md` file produce last-writer-wins corruption. Reddit usage data shows parallel agents on the same worktree is a real pattern (e.g. 8 parallel code-review sub-agents, parallel feature agents). + +### Decision + +Add a second, outer lock layer: `crossProcessLock.ts` implements a cross-process advisory lockfile at `.memory/.lock`. All writes first acquire this lock, then fall through to the existing per-file in-process lock. + +Key choices: + +- **One directory-level lock, not per-file lockfiles.** Write frequency is low. A single `.lock` file avoids proliferating 5+ lockfiles and is simpler to reason about. +- **Pure Node.js — no runtime dependencies.** `fs.open(path, 'wx')` is the atomic exclusive-create primitive; it is one line. The extension has zero runtime npm dependencies and must stay that way. +- **Stale detection: PID liveness + age fallback.** `process.kill(pid, 0)` checks PID existence without sending a signal. If the PID is gone the lock is stale. Age > 10 s is a fallback for cross-machine or PID-reuse edge cases. +- **`clearStaleLockOnStartup()` called from `ensureMemoryDir`.** Cleans up any lock file left behind by a crashed process on the first write of a new session. + +### Alternatives Rejected + +- `proper-lockfile` npm package: correct, but adds a runtime dependency. Not worth it given the core primitive is trivial to implement. +- Per-file lockfiles: more granular but multiplies file clutter and adds complexity for no practical benefit at current write rates. + +### Consequences + +- Parallel agents writing memories to the same worktree are serialised. No data loss. +- Zero new npm dependencies. +- The `.lock` file is visible in the `.memory/` folder during writes. It is ephemeral (removed on release) and should be added to `.gitignore` by projects that track `.memory/`. +- `withMemoryWriteLock(filePath, fn)` in `markdownStore.ts` is the only call site — neither lock is accessible outside this module. diff --git a/docs/getting-started.md b/docs/getting-started.md index 00fcc62..186ebfa 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,7 +11,9 @@ Open VS Code. Press `Ctrl+Shift+X` to open Extensions. Search for **HackLM Memor Or click one of these links: - [Install from VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=hacklm.hacklm-memory) -- [Install from Open VSX](https://open-vsx.org/extension/hacklm/hacklm-memory) *(for Google Antigravity, Gitpod, and other VS Code-based IDEs)* +- [Install from Open VSX](https://open-vsx.org/extension/hacklm/hacklm-memory) + +> **Using an Open VSX-compatible editor?** The extension installs and sets up the `.memory/` files and `copilot-instructions.md` reference block in any editor. The `storeMemory` and `queryMemory` LM tools, tree view, and status bar require the VS Code LM Tool API and are only available in VS Code 1.99+. See the [compatibility table](../README.md#open-vsx-editors) for details. --- diff --git a/docs/index.md b/docs/index.md index bd46001..93ec387 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,7 +18,7 @@ Free. No cloud. No sign-up. Works in VS Code out of the box. [Get started →](getting-started.md) -Also on [Open VSX](https://open-vsx.org/extension/hacklm/hacklm-memory) for Google Antigravity, Gitpod, and other VS Code-based IDEs. +Also on [Open VSX](https://open-vsx.org/extension/hacklm/hacklm-memory). Support for additional editors is [planned](roadmap.md). --- diff --git a/docs/roadmap.md b/docs/roadmap.md index 08a3917..f3aef2e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,10 +4,8 @@ HackLM Memory is a fully functional VS Code extension providing persistent long-term memory for GitHub Copilot. The core pipeline — store, query, dedup, score, cleanup, gap analysis — is complete and working. -Supported editors (via Open VSX): +Supported editors: - VS Code 1.99+ -- Google Antigravity (Open VSX-compatible) -- Gitpod (Open VSX-compatible) --- @@ -23,7 +21,7 @@ Start in `extension/src/tools/`. ### LM API Compatibility Matrix -We don't yet have a systematic record of which Open VSX-compatible editors fully implement the VS Code LM Tool API (`vscode.lm.registerTool`, `vscode.lm.selectChatModels`). Contributions testing HackLM Memory in Google Antigravity and Gitpod — and documenting results — would help users know what to expect. +VS Code 1.99+ is confirmed working. Other Open VSX-compatible editors are untested. Contributions testing HackLM Memory in additional editors and documenting results are welcome. ### Export / Import @@ -53,19 +51,34 @@ Start in `extension/src/utils.ts` and `extension/package.json`. ## Planned Future Work +### Additional Editor Support (MCP-based) + +**Status:** Not started. Architecture decided — MCP-based. + +Google Antigravity supports MCP (Model Context Protocol), which allows tools to be called by the agent without the VS Code extension API. The plan: + +1. Extract the storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) into a shared `packages/storage` package (plain TypeScript/Node, no VS Code dependency). +2. Build an `mcp/` package — an MCP server that implements `storeMemory` and `queryMemory` tools backed by the shared storage layer. +3. The MCP server accepts `workspaceRoot` as a tool parameter rather than reading from VS Code context. +4. LM-based deduplication (redundancy check) is omitted from the MCP server — Jaccard fuzzy matching alone is used. No VS Code LM API equivalent exists in this context. +5. Gap analysis / session review is omitted from the MCP server — no user prompt UI is available. +6. Antigravity wires it up via `mcp_config.json`; a `.agent/skills/memory/SKILL.md` tells the agent when to call the tools. + +The `.memory/*.md` file format is the stable, editor-agnostic contract. Any implementation must read/write that format correctly for memory to be shareable across editors. + +See [ADR 0017](decisions.md#0017--additional-editor-support-via-mcp) for the architectural decision record. + ### JetBrains Support **Status:** Not started. Deferred until a maintainer or contributor with JetBrains plugin experience volunteers. What this requires: -1. Extract the storage layer (`dedup.ts`, `scoring.ts`, `search.ts`, `markdownStore.ts`) into a shared `core/` package (plain TypeScript/Node, no VS Code dependency). -2. Write a separate Kotlin/Java IntelliJ Platform plugin that implements the same `.memory/*.md` file format (spec in [api-reference.md](api-reference.md)) and integrates with JetBrains AI Assistant's tool/agent API. +1. The `packages/storage` extraction from the Antigravity work above (prerequisite). +2. A separate Kotlin/Java IntelliJ Platform plugin that implements the same `.memory/*.md` file format (spec in [api-reference.md](api-reference.md)) and integrates with JetBrains AI Assistant's tool/agent API. 3. Publish the JetBrains plugin to [plugins.jetbrains.com](https://plugins.jetbrains.com/). -The `.memory/*.md` file format is the stable, editor-agnostic contract. Any implementation must read/write that format correctly for memory to be shareable across editors. - See [ADR 0016](decisions.md#0016--vs-code-first-jetbrains-deferred) for the architectural decision record. ### Neovim / Other Editors -Similar to JetBrains — requires a separate plugin implementation using the relevant editor's extension API (nvim-lspconfig, etc.). The `core/` extraction above is a prerequisite. +Similar to JetBrains — requires a separate plugin implementation using the relevant editor's extension API. The `packages/storage` extraction above is a prerequisite. diff --git a/docs/user-guide.md b/docs/user-guide.md index c2eac6a..245da62 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -111,11 +111,11 @@ All settings are under `hacklm-memory.*`. Open them in **Settings UI** (`Ctrl+,` | `autoCleanupFrequency` | `10` | Global | Auto-cleanup after every N store operations | | `autoApproveStore` | `false` | Global | Skip the confirmation prompt when saving memories | | `manageInstructionFile` | `true` | Global | Allow the extension to manage the `` block in `.github/copilot-instructions.md` | -| `categoryLimit.Instruction` | `15` | Workspace | Max entries in the Instruction category | -| `categoryLimit.Quirk` | `20` | Workspace | Max entries in the Quirk category | -| `categoryLimit.Preference` | `20` | Workspace | Max entries in the Preference category | -| `categoryLimit.Decision` | `20` | Workspace | Max entries in the Decision category | -| `categoryLimit.Security` | `15` | Workspace | Max entries in the Security category | +| `categoryLimit.Instruction` | `30` | Workspace | Max entries in the Instruction category | +| `categoryLimit.Quirk` | `40` | Workspace | Max entries in the Quirk category | +| `categoryLimit.Preference` | `40` | Workspace | Max entries in the Preference category | +| `categoryLimit.Decision` | `40` | Workspace | Max entries in the Decision category | +| `categoryLimit.Security` | `30` | Workspace | Max entries in the Security category | --- diff --git a/extension/README.md b/extension/README.md index a1fe304..22991f4 100644 --- a/extension/README.md +++ b/extension/README.md @@ -1,7 +1,7 @@ # HackLM Memory

- HackLM Memory + HackLM Memory

@@ -65,7 +65,7 @@ HackLM Memory gives Copilot a persistent memory across sessions. It stores decis | `hacklm-memory.autoApproveStore` | `false` | Skip confirmation when saving memories | | `hacklm-memory.manageInstructionFile` | `true` | Manage `.github/copilot-instructions.md` automatically | | `hacklm-memory.autoCleanupFrequency` | `10` | Run cleanup automatically every N store operations | -| `hacklm-memory.categoryLimit.*` | `20` | Max entries per category | +| `hacklm-memory.categoryLimit.*` | `40` | Max entries per category | --- diff --git a/extension/package.json b/extension/package.json index 997e080..bf31f8d 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "hacklm-memory", "displayName": "HackLM Memory", "description": "Persistent memory layer for VS Code Copilot — learn from interactions, retain insights, improve over time", - "version": "1.0.0", + "version": "1.1.0", "publisher": "hacklm", "icon": "icon.png", "license": "GPL-3.0-only", @@ -289,37 +289,37 @@ }, "hacklm-memory.categoryLimit.Instruction": { "type": "number", - "default": 15, + "default": 30, "minimum": 1, - "maximum": 50, + "maximum": 100, "markdownDescription": "Max entries to keep in the `Instruction` category." }, "hacklm-memory.categoryLimit.Quirk": { "type": "number", - "default": 20, + "default": 40, "minimum": 1, - "maximum": 50, + "maximum": 100, "markdownDescription": "Max entries to keep in the `Quirk` category." }, "hacklm-memory.categoryLimit.Preference": { "type": "number", - "default": 20, + "default": 40, "minimum": 1, - "maximum": 50, + "maximum": 100, "markdownDescription": "Max entries to keep in the `Preference` category." }, "hacklm-memory.categoryLimit.Decision": { "type": "number", - "default": 20, + "default": 40, "minimum": 1, - "maximum": 50, + "maximum": 100, "markdownDescription": "Max entries to keep in the `Decision` category." }, "hacklm-memory.categoryLimit.Security": { "type": "number", - "default": 15, + "default": 30, "minimum": 1, - "maximum": 50, + "maximum": 100, "markdownDescription": "Max entries to keep in the `Security` category." }, "hacklm-memory.autoApproveStore": { diff --git a/extension/src/extension.ts b/extension/src/extension.ts index ab8abfb..078fc14 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -1,5 +1,5 @@ import * as vscode from 'vscode'; -import { generateInstructionFiles, ensureMemoryFiles } from './instructionFiles'; +import { generateInstructionFiles, ensureMemoryFiles, patchPlanAgent } from './instructionFiles'; import { createStatusBar, updateStatusBar } from './statusBar'; import { showMemoryList, deleteMemoryInteractive, openMemoryFolder } from './storage/markdownStore'; import { showMemoryPanel, showMemoryStats, runCleanupInteractive } from './memoryPanel'; @@ -47,6 +47,8 @@ export async function activate(context: vscode.ExtensionContext): Promise } } + await patchPlanAgent(context); + const statusBar = createStatusBar(context); context.subscriptions.push(statusBar); @@ -72,6 +74,7 @@ export async function activate(context: vscode.ExtensionContext): Promise vscode.commands.registerCommand('hacklm-memory.open', openMemoryFolder), vscode.commands.registerCommand('hacklm-memory.reinit', async () => { await generateInstructionFiles(workspaceFolder); + await patchPlanAgent(context); vscode.window.showInformationMessage('HackLM Memory: Instruction files regenerated.'); }), vscode.commands.registerCommand('hacklm-memory.cleanup', runCleanupInteractive), diff --git a/extension/src/instructionFiles.ts b/extension/src/instructionFiles.ts index f56b18d..d7f36c4 100644 --- a/extension/src/instructionFiles.ts +++ b/extension/src/instructionFiles.ts @@ -1,10 +1,14 @@ import * as vscode from 'vscode'; import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { getOutputChannel } from './outputChannel'; const MEMORY_MARKER_START = ''; const MEMORY_MARKER_END = ''; +/** Tools added to the Copilot Plan agent on every activation. */ +const PLAN_AGENT_MEMORY_TOOLS = ['queryMemory', 'storeMemory'] as const; + function getCopilotInstructionsSection(): string { return `${MEMORY_MARKER_START} ## Memory-Augmented Context @@ -76,6 +80,75 @@ async function upsertManagedSection( await fs.writeFile(filePath, content, 'utf-8'); } +function getPlanAgentMemorySection(): string { + return `${MEMORY_MARKER_START} +## Memory Context + +Call \`#queryMemory\` at the start of every planning session to retrieve relevant project decisions, conventions, and constraints before researching or drafting. + +Call \`#storeMemory\` when the planning discussion surfaces a new architectural decision or constraint not yet in memory. Store it **before presenting the plan**, while the rationale is still in context. + +Memory categories most relevant during planning: +- **Security** — constraints the plan must never violate (always query first) +- **Decision** — prior architectural commitments the plan must respect +- **Instruction** — how the team expects work to be approached +${MEMORY_MARKER_END}`; +} + +/** + * Patches the Copilot Chat built-in Plan agent to include memory tools. + * Runs on every activation — idempotent. Silent no-op if Copilot Chat is not installed. + */ +export async function patchPlanAgent(context: vscode.ExtensionContext): Promise { + const planAgentPath = path.join( + context.globalStorageUri.fsPath, + '..', + 'github.copilot-chat', + 'plan-agent', + 'Plan.agent.md' + ); + + let content: string; + try { + content = await fs.readFile(planAgentPath, 'utf-8'); + } catch { + // Copilot Chat not installed or Plan agent absent — silent no-op + return; + } + + let patched = content; + + // Add memory tools to the tools: array if not already present + const toolsLineMatch = /^tools: \[(.+)\]$/m.exec(patched); + if (toolsLineMatch) { + const existing = toolsLineMatch[1]; + const toolsToAdd = PLAN_AGENT_MEMORY_TOOLS.filter(t => !existing.includes(`'${t}'`)); + if (toolsToAdd.length > 0) { + const addition = toolsToAdd.map(t => `'${t}'`).join(', '); + patched = patched.replace( + toolsLineMatch[0], + `tools: [${existing}, ${addition}]` + ); + } + } + + // Inject/update the memory instructions section in the agent body + const section = getPlanAgentMemorySection(); + if (patched.includes(MEMORY_MARKER_START) && patched.includes(MEMORY_MARKER_END)) { + const startIdx = patched.indexOf(MEMORY_MARKER_START); + const endIdx = patched.indexOf(MEMORY_MARKER_END) + MEMORY_MARKER_END.length; + patched = patched.substring(0, startIdx) + section + patched.substring(endIdx); + } else { + if (!patched.endsWith('\n')) { patched += '\n'; } + patched += '\n' + section + '\n'; + } + + if (patched !== content) { + await fs.writeFile(planAgentPath, patched, 'utf-8'); + getOutputChannel().appendLine('[HackLM Memory] Patched Plan agent with memory tools.'); + } +} + export async function generateInstructionFiles(workspaceFolder: vscode.WorkspaceFolder): Promise { const root = workspaceFolder.uri.fsPath; const copilotInstructionsPath = path.join(root, '.github', 'copilot-instructions.md'); diff --git a/extension/src/memoryPanel.ts b/extension/src/memoryPanel.ts index 5aac93d..3859a4b 100644 --- a/extension/src/memoryPanel.ts +++ b/extension/src/memoryPanel.ts @@ -164,7 +164,7 @@ async function showSettings(): Promise { const selected = await vscode.window.showQuickPick( categories.map(c => { const limitKey = `categoryLimit.${c}`; - return { label: c, description: `Current limit: ${config.get(limitKey, 20)}`, category: c }; + return { label: c, description: `Current limit: ${config.get(limitKey, 40)}`, category: c }; }), { placeHolder: 'Select category to configure' } ); @@ -176,7 +176,7 @@ async function showSettings(): Promise { value: String(config.get(`categoryLimit.${selected.category}`, 20)), validateInput: (v) => { const num = Number.parseInt(v); - return num > 0 && num <= 50 ? null : 'Must be between 1 and 50'; + return num > 0 && num <= 100 ? null : 'Must be between 1 and 100'; }, }); diff --git a/extension/src/storage/crossProcessLock.test.ts b/extension/src/storage/crossProcessLock.test.ts new file mode 100644 index 0000000..c72d03e --- /dev/null +++ b/extension/src/storage/crossProcessLock.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import * as crypto from 'node:crypto'; +import { + withCrossProcessLock, + isStale, + clearStaleLockOnStartup, + STALE_MS, +} from './crossProcessLock'; + +const LOCK_FILE = '.lock'; + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function makeTempDir(): Promise { + const dir = path.join(os.tmpdir(), `cpl-test-${crypto.randomUUID()}`); + await fs.mkdir(dir, { recursive: true }); + return dir; +} + +// ─── helpers ──────────────────────────────────────────────────────────────── + +/** + * A racy read-modify-write: reads a file, sleeps (forcing all parallel callers + * to observe the same stale state), then writes back with the entry appended. + * Without a lock, the final file contains only the last writer's view. + */ +async function racingWrite(filePath: string, entry: string): Promise { + let text = ''; + try { text = await fs.readFile(filePath, 'utf-8'); } catch { /* empty file is fine */ } + const lines = text ? text.split('\n').filter(Boolean) : []; + await sleep(10); // ensures all readers finish before any writer starts + lines.push(entry); + await fs.writeFile(filePath, lines.join('\n'), 'utf-8'); +} + +// ─── test suite ───────────────────────────────────────────────────────────── + +describe('crossProcessLock', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + try { await fs.rm(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } + }); + + async function useTempDir(): Promise { + const dir = await makeTempDir(); + tempDirs.push(dir); + return dir; + } + + // ── Group 1: prove the bug ────────────────────────────────────────────── + + describe('without a lock: data is lost', () => { + it('last writer wins when concurrent writes race without a lock', async () => { + const dir = await useTempDir(); + const filePath = path.join(dir, 'test.md'); + await fs.writeFile(filePath, '', 'utf-8'); + + const writers = ['a', 'b', 'c', 'd', 'e']; + await Promise.all(writers.map(entry => racingWrite(filePath, entry))); + + const text = await fs.readFile(filePath, 'utf-8'); + const lines = text.split('\n').filter(Boolean); + + // All writers read the same empty state, then all overwrite each other. + // The result is fewer than 5 entries — proving data loss. + expect(lines.length).toBeLessThan(writers.length); + }); + }); + + // ── Group 2: prove the fix ────────────────────────────────────────────── + + describe('withCrossProcessLock: all writes land', () => { + it('serialises concurrent writers so no entry is lost', async () => { + const dir = await useTempDir(); + const filePath = path.join(dir, 'test.md'); + await fs.writeFile(filePath, '', 'utf-8'); + + const writers = ['a', 'b', 'c', 'd', 'e']; + await Promise.all( + writers.map(entry => + withCrossProcessLock(dir, () => racingWrite(filePath, entry)) + ) + ); + + const text = await fs.readFile(filePath, 'utf-8'); + const lines = text.split('\n').filter(Boolean); + + expect(lines.length).toBe(writers.length); + expect([...lines].sort()).toEqual([...writers].sort()); + }); + + it('removes the lock file after successful completion', async () => { + const dir = await useTempDir(); + await withCrossProcessLock(dir, async () => { /* no-op */ }); + const lockPath = path.join(dir, LOCK_FILE); + await expect(fs.access(lockPath)).rejects.toThrow(); + }); + + it('removes the lock file even when fn throws', async () => { + const dir = await useTempDir(); + const lockPath = path.join(dir, LOCK_FILE); + await expect( + withCrossProcessLock(dir, async () => { throw new Error('boom'); }) + ).rejects.toThrow('boom'); + await expect(fs.access(lockPath)).rejects.toThrow(); + }); + }); + + // ── Group 3: stale lock recovery ──────────────────────────────────────── + + describe('stale lock recovery', () => { + it('removes a stale lock (dead PID + old timestamp) and proceeds', async () => { + const dir = await useTempDir(); + const lockPath = path.join(dir, LOCK_FILE); + await fs.writeFile( + lockPath, + JSON.stringify({ pid: 99999999, ts: Date.now() - (STALE_MS + 1000) }), + 'utf-8' + ); + + let ran = false; + await withCrossProcessLock(dir, async () => { ran = true; }); + expect(ran).toBe(true); + }); + + it('clearStaleLockOnStartup removes a stale lock', async () => { + const dir = await useTempDir(); + const lockPath = path.join(dir, LOCK_FILE); + await fs.writeFile( + lockPath, + JSON.stringify({ pid: 99999999, ts: Date.now() - (STALE_MS + 1000) }), + 'utf-8' + ); + + await clearStaleLockOnStartup(dir); + await expect(fs.access(lockPath)).rejects.toThrow(); + }); + + it('clearStaleLockOnStartup leaves a fresh live lock untouched', async () => { + const dir = await useTempDir(); + const lockPath = path.join(dir, LOCK_FILE); + await fs.writeFile( + lockPath, + JSON.stringify({ pid: process.pid, ts: Date.now() }), + 'utf-8' + ); + + await clearStaleLockOnStartup(dir); + // Lock should still be present — this process is alive and the lock is fresh + await expect(fs.access(lockPath)).resolves.not.toThrow(); + }); + + it('clearStaleLockOnStartup is a no-op when no lock file exists', async () => { + const dir = await useTempDir(); + await expect(clearStaleLockOnStartup(dir)).resolves.not.toThrow(); + }); + }); + + // ── Group 4: isStale unit tests ───────────────────────────────────────── + + describe('isStale', () => { + it('returns true for a non-existent PID', () => { + expect(isStale({ pid: 99999999, ts: Date.now() })).toBe(true); + }); + + it('returns false for the current process with a fresh timestamp', () => { + expect(isStale({ pid: process.pid, ts: Date.now() })).toBe(false); + }); + + it('returns true when age exceeds STALE_MS even if PID is alive', () => { + expect(isStale({ pid: process.pid, ts: Date.now() - (STALE_MS + 1) })).toBe(true); + }); + + it('returns true for a dead PID with a fresh timestamp', () => { + // No process with PID 99999999 — dead PID alone is sufficient + expect(isStale({ pid: 99999999, ts: Date.now() })).toBe(true); + }); + }); +}); diff --git a/extension/src/storage/crossProcessLock.ts b/extension/src/storage/crossProcessLock.ts new file mode 100644 index 0000000..08e4b75 --- /dev/null +++ b/extension/src/storage/crossProcessLock.ts @@ -0,0 +1,119 @@ +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +const LOCK_FILE = '.lock'; + +/** How long before a lock is considered abandoned (ms). */ +export const STALE_MS = 10_000; + +/** Max attempts to acquire the lock before giving up. */ +const MAX_RETRIES = 20; + +/** Base delay between retries — multiplied by attempt number (linear back-off). */ +const RETRY_BASE_MS = 50; + +interface LockContent { + pid: number; + ts: number; +} + +/** + * Returns true when a lock entry should be considered abandoned. + * Two conditions either of which is sufficient: + * 1. Age > STALE_MS — the process holding it has been gone long enough. + * 2. PID is not alive — verified via process.kill(pid, 0). + */ +export function isStale(content: LockContent): boolean { + if (Date.now() - content.ts > STALE_MS) { return true; } + try { + // Sends signal 0: no signal delivered, but OS checks if PID exists. + // Throws with code ESRCH when the process does not exist. + process.kill(content.pid, 0); + return false; // process is alive + } catch { + return true; // ESRCH: process not found + } +} + +async function readLockContent(lockPath: string): Promise { + try { + const text = await fs.readFile(lockPath, 'utf-8'); + return JSON.parse(text) as LockContent; + } catch { + return null; + } +} + +/** + * Acquires an advisory lockfile in `memoryDir`. + * Returns a release function that removes the lock file. + * + * Uses `fs.open(path, 'wx')` — exclusive create — which is atomic on all + * POSIX filesystems and NTFS. Only one caller can create the file; all others + * get EEXIST and must wait or detect staleness. + */ +async function acquireLock(memoryDir: string): Promise<() => Promise> { + const lockPath = path.join(memoryDir, LOCK_FILE); + + for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + const fh = await fs.open(lockPath, 'wx'); + await fh.writeFile(JSON.stringify({ pid: process.pid, ts: Date.now() }), 'utf-8'); + await fh.close(); + return async () => { + try { await fs.unlink(lockPath); } catch { /* ignore — already removed */ } + }; + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') { + throw err; // unexpected error — propagate + } + + // Lock file exists — check if the holder is still alive + const content = await readLockContent(lockPath); + if (content && isStale(content)) { + try { await fs.unlink(lockPath); } catch { /* another process removed it first */ } + continue; // retry immediately without sleeping + } + + // Holder is alive — wait with linear back-off + await new Promise(resolve => setTimeout(resolve, RETRY_BASE_MS * (attempt + 1))); + } + } + + throw new Error( + `[crossProcessLock] Could not acquire lock after ${MAX_RETRIES} retries: ${lockPath}` + ); +} + +/** + * Acquires a directory-level advisory lock, runs `fn`, then releases the lock. + * Safe across processes: uses an exclusive lockfile rather than an in-process Map. + */ +export async function withCrossProcessLock( + memoryDir: string, + fn: () => Promise +): Promise { + const release = await acquireLock(memoryDir); + try { + return await fn(); + } finally { + await release(); + } +} + +/** + * Called at extension startup — cleans up a stale lockfile left behind by a + * crashed process so that the extension does not block indefinitely on first use. + * Non-fatal: all errors are silently ignored. + */ +export async function clearStaleLockOnStartup(memoryDir: string): Promise { + try { + const lockPath = path.join(memoryDir, LOCK_FILE); + const content = await readLockContent(lockPath); + if (content && isStale(content)) { + await fs.unlink(lockPath); + } + } catch { + // Non-fatal + } +} diff --git a/extension/src/storage/markdownStore.ts b/extension/src/storage/markdownStore.ts index 8b42ef0..64bbbdb 100644 --- a/extension/src/storage/markdownStore.ts +++ b/extension/src/storage/markdownStore.ts @@ -1,10 +1,13 @@ import * as vscode from 'vscode'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import { withCrossProcessLock, clearStaleLockOnStartup } from './crossProcessLock'; /** - * Per-file write locks — prevents concurrent read-then-write races when - * two operations (e.g. store + background cleanup) touch the same file. + * Per-file in-process locks — serialises concurrent calls within the same + * extension host process. Combined with the cross-process lock below, this + * gives two-tier protection: outer = cross-process lockfile, inner = per-file + * promise queue. */ const _fileLocks = new Map>(); @@ -24,6 +27,15 @@ async function withFileLock(filePath: string, fn: () => Promise): Promise< } } +/** + * Two-tier write lock: acquires the cross-process directory lock first, then + * the in-process per-file lock. Callers outside this module never call + * withFileLock or withCrossProcessLock directly. + */ +async function withMemoryWriteLock(filePath: string, fn: () => Promise): Promise { + return withCrossProcessLock(path.dirname(filePath), () => withFileLock(filePath, fn)); +} + export const CATEGORY_FILES: Record = { 'Instruction': 'instructions.md', 'Quirk': 'quirks.md', @@ -51,6 +63,7 @@ export function getMemoryDir(): string { export async function ensureMemoryDir(): Promise { const memDir = getMemoryDir(); await fs.mkdir(memDir, { recursive: true }); + await clearStaleLockOnStartup(memDir); for (const [category, filename] of Object.entries(CATEGORY_FILES)) { const filePath = path.join(memDir, filename); @@ -76,7 +89,7 @@ export async function appendMemory(category: string, content: string): Promise fs.appendFile(filePath, `\n- ${content.trim()}\n`, 'utf-8')); + await withMemoryWriteLock(filePath, () => fs.appendFile(filePath, `\n- ${content.trim()}\n`, 'utf-8')); } export async function upsertMemory( @@ -95,7 +108,7 @@ export async function upsertMemory( const filePath = path.join(getMemoryDir(), filename); const bulletLine = `- [${slug}] ${content.trim()}`; - return withFileLock(filePath, async () => { + return withMemoryWriteLock(filePath, async () => { let text = ''; try { text = await fs.readFile(filePath, 'utf-8'); @@ -125,7 +138,7 @@ export async function migrateFiles(): Promise { const text = await fs.readFile(filePath, 'utf-8'); const migrated = stripDateHeaders(text); if (migrated !== text) { - await withFileLock(filePath, () => fs.writeFile(filePath, migrated, 'utf-8')); + await withMemoryWriteLock(filePath, () => fs.writeFile(filePath, migrated, 'utf-8')); } } catch { // skip missing files @@ -212,7 +225,7 @@ export async function deleteMemory( if (!filename) { return false; } const filePath = path.join(getMemoryDir(), filename); - return withFileLock(filePath, async () => { + return withMemoryWriteLock(filePath, async () => { try { const text = await fs.readFile(filePath, 'utf-8'); const lines = stripDateHeaders(text).split('\n'); diff --git a/extension/src/utils.ts b/extension/src/utils.ts index 8a1186e..111c324 100644 --- a/extension/src/utils.ts +++ b/extension/src/utils.ts @@ -2,5 +2,5 @@ import * as vscode from 'vscode'; export function getEffectiveLimit(category: string): number { const config = vscode.workspace.getConfiguration('hacklm-memory'); - return config.get(`categoryLimit.${category}`, 20); + return config.get(`categoryLimit.${category}`, 40); }