Add multi-directory support for stores

Stores can now index documents from multiple directories by specifying
--docs multiple times: rag store new my-store --docs ./path1 --docs ./path2

Changes:
- CLI: --docs flag accepts multiple paths via action="append"
- store.py: docs_dir -> docs_dirs (list), with backwards compat
- ingest.py: load_documents_from_dirs() aggregates from all paths
- migrate.py: passes docs_dir as single-element list to create_store
- All tests updated for new list-based interface

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: JoshuaRamirez

src/rag/cli.py

@@ -100,18 +100,24 @@ def cmd_store_new(args: argparse.Namespace) -> int:
     from pathlib import Path
 
     name = args.name
-    docs_path = args.docs
+    docs_paths = args.docs  # List of paths or None
 
     # Prompt for docs path if not provided
-    if not docs_path:
-        docs_path = console.input(f"[bold]Documents path for '{name}':[/bold] ").strip()
-        if not docs_path:
+    if not docs_paths:
+        docs_input = console.input(f"[bold]Documents path for '{name}':[/bold] ").strip()
+        if not docs_input:
             console.print("[red]Error: Documents path is required.[/red]")
             return 1
+        docs_paths = [docs_input]
+
+    # Convert to Path objects
+    docs_dirs = [Path(p) for p in docs_paths]
 
     try:
-        create_store(name, Path(docs_path))
+        create_store(name, docs_dirs)
         console.print(f"[green]Store '{name}' created successfully.[/green]")
+        if len(docs_dirs) > 1:
+            console.print(f"[dim]Indexing {len(docs_dirs)} directories.[/dim]")
 
         # Set as default if it's the first store
         stores = list_stores()
@@ -138,7 +144,7 @@ def cmd_store_list(args: argparse.Namespace) -> int:
     table = Table(title="Vector Bot Stores")
     table.add_column("Name", style="cyan")
     table.add_column("Default", style="green")
-    table.add_column("Documents Path", style="dim")
+    table.add_column("Documents", style="dim")
     table.add_column("Last Indexed", style="dim")
     table.add_column("Chunks", style="dim", justify="right")
 
@@ -157,10 +163,19 @@ def cmd_store_list(args: argparse.Namespace) -> int:
         chunk_count = store.get("chunk_count")
         chunks_str = str(chunk_count) if chunk_count is not None else "-"
 
+        # Format docs_dirs for display
+        docs_dirs = store.get("docs_dirs", [])
+        if len(docs_dirs) == 0:
+            docs_display = "-"
+        elif len(docs_dirs) == 1:
+            docs_display = docs_dirs[0]
+        else:
+            docs_display = f"{len(docs_dirs)} directories"
+
         table.add_row(
             store["name"],
             is_default,
-            store.get("docs_dir", "-"),
+            docs_display,
             last_indexed,
             chunks_str,
         )
@@ -183,7 +198,17 @@ def cmd_store_info(args: argparse.Namespace) -> int:
 
     console.print(f"[bold]Store: {name}[/bold]")
     console.print(f"  Default: {'yes' if is_default else 'no'}")
-    console.print(f"  Documents path: {store.get('docs_dir', '-')}")
+
+    # Display document directories
+    docs_dirs = store.get("docs_dirs", [])
+    if len(docs_dirs) == 0:
+        console.print("  Documents: -")
+    elif len(docs_dirs) == 1:
+        console.print(f"  Documents: {docs_dirs[0]}")
+    else:
+        console.print(f"  Documents: {len(docs_dirs)} directories")
+        for d in docs_dirs:
+            console.print(f"    - {d}")
 
     index_dir = get_store_index_dir(name)
     console.print(f"  Index path: {index_dir}")
@@ -338,7 +363,8 @@ def cmd_migrate(args: argparse.Namespace) -> int:
         console.print()
         console.print("[bold]New store info:[/bold]")
         console.print(f"  Name: {store_config['name']}")
-        console.print(f"  Documents: {store_config['docs_dir']}")
+        docs_dirs = store_config.get('docs_dirs', [])
+        console.print(f"  Documents: {docs_dirs[0] if docs_dirs else '-'}")
         console.print(f"  Index: {get_store_index_dir(store_name)}")
         console.print(f"  Set as default: yes")
     except (ValueError, FileNotFoundError) as e:
@@ -472,7 +498,8 @@ def main(argv: Optional[List[str]] = None) -> int:
     store_new_parser.add_argument(
         "--docs",
         metavar="PATH",
-        help="Path to documents directory (prompts if not given)",
+        action="append",
+        help="Path to documents directory (can be specified multiple times)",
         default=None,
     )
 

src/rag/ingest.py

@@ -88,25 +88,25 @@ def setup_llm_settings() -> None:
 
 
 def load_documents(docs_dir: Path) -> List:
-    """Load documents from the specified directory."""
+    """Load documents from a single directory."""
     if not docs_dir.exists():
         docs_dir.mkdir(parents=True, exist_ok=True)
         console.print(f"[yellow]Created docs directory: {docs_dir}[/yellow]")
         return []
-    
+
     # Define file extensions to load
     extensions = [".txt", ".md", ".pdf", ".json", ".csv"]
-    
+
     # Check for documents
     doc_files: list[Path] = []
     for ext in extensions:
         doc_files.extend(docs_dir.glob(f"*{ext}"))
         doc_files.extend(docs_dir.glob(f"**/*{ext}"))
-    
+
     if not doc_files:
         console.print(f"[yellow]No documents found in {docs_dir}[/yellow]")
         return []
-    
+
     # Filter out large files (>20MB)
     valid_files: list[Path] = []
     for file in doc_files:
@@ -115,27 +115,47 @@ def load_documents(docs_dir: Path) -> List:
             console.print(f"[yellow]Skipping large file (>{size_mb:.1f}MB): {file.name}[/yellow]")
         else:
             valid_files.append(file)
-    
+
     if not valid_files:
         console.print("[yellow]No valid documents to index[/yellow]")
         return []
-    
-    console.print(f"[blue]Loading {len(valid_files)} documents...[/blue]")
-    
+
+    console.print(f"[blue]Loading {len(valid_files)} documents from {docs_dir}...[/blue]")
+
     # Load documents
     reader = SimpleDirectoryReader(
         input_dir=str(docs_dir),
         recursive=True,
         exclude_hidden=True,
         required_exts=extensions,
     )
-    
+
     documents = reader.load_data()
     console.print(f"[green]Loaded {len(documents)} document chunks[/green]")
-    
+
     return documents
 
 
+def load_documents_from_dirs(docs_dirs: List[Path]) -> List:
+    """Load documents from multiple directories.
+
+    Args:
+        docs_dirs: List of paths to document directories.
+
+    Returns:
+        Combined list of documents from all directories.
+    """
+    all_documents = []
+    for docs_dir in docs_dirs:
+        documents = load_documents(docs_dir)
+        all_documents.extend(documents)
+
+    if len(docs_dirs) > 1:
+        console.print(f"[blue]Total: {len(all_documents)} chunks from {len(docs_dirs)} directories[/blue]")
+
+    return all_documents
+
+
 def ingest(
     store_name: str | None = None,
     verbose: bool = False,
@@ -155,11 +175,14 @@ def ingest(
     store_name = resolve_store(store_name)
     store = get_store(store_name)
 
-    docs_dir = Path(store["docs_dir"])
+    # Get document directories (handles backwards compatibility)
+    docs_dirs = [Path(d) for d in store.get("docs_dirs", [])]
     index_dir = get_store_index_dir(store_name)
 
     console.print("[bold]Document Ingestion[/bold]")
     console.print(f"[blue]Store: {store_name}[/blue]")
+    if len(docs_dirs) > 1:
+        console.print(f"[dim]Indexing {len(docs_dirs)} directories[/dim]")
 
     # Setup LLM settings
     setup_llm_settings()
@@ -194,10 +217,11 @@ def ingest(
     if force and index_dir.exists():
         console.print("[yellow]Force flag set - rebuilding index...[/yellow]")
 
-    # Load documents
-    documents = load_documents(docs_dir)
+    # Load documents from all directories
+    documents = load_documents_from_dirs(docs_dirs)
     if not documents:
-        raise RuntimeError(f"No documents found in {docs_dir}")
+        dirs_str = ", ".join(str(d) for d in docs_dirs)
+        raise RuntimeError(f"No documents found in: {dirs_str}")
 
     # Count unique source files from loaded documents
     source_files = set()

src/rag/migrate.py

@@ -133,8 +133,8 @@ def migrate_legacy_index(
     if not docs_dir.is_absolute():
         docs_dir = docs_dir.resolve()
 
-    # Create the new store
-    store_config = create_store(store_name, docs_dir)
+    # Create the new store (pass docs_dir as a list)
+    store_config = create_store(store_name, [docs_dir])
 
     # Copy index files from legacy to new store
     new_index_dir = get_store_index_dir(store_name)

src/rag/store.py

@@ -131,7 +131,7 @@ def list_stores() -> list[dict[str, Any]]:
     Returns:
         List of store info dictionaries, each containing:
         - name: Store name
-        - docs_dir: Path to documents directory
+        - docs_dirs: List of paths to documents directories
         - last_indexed: ISO timestamp of last indexing, or None
         - chunk_count: Number of indexed chunks
     """
@@ -153,9 +153,16 @@ def list_stores() -> list[dict[str, Any]]:
             with open(store_config_path, "r", encoding="utf-8") as f:
                 store_data = json.load(f)
 
+            # Handle both old (docs_dir) and new (docs_dirs) formats
+            docs_dirs = store_data.get("docs_dirs")
+            if docs_dirs is None:
+                # Backwards compatibility: convert single docs_dir to list
+                old_docs_dir = store_data.get("docs_dir")
+                docs_dirs = [old_docs_dir] if old_docs_dir else []
+
             stores.append({
                 "name": store_data.get("name", store_dir.name),
-                "docs_dir": store_data.get("docs_dir"),
+                "docs_dirs": docs_dirs,
                 "last_indexed": store_data.get("last_indexed"),
                 "chunk_count": store_data.get("chunk_count", 0),
             })
@@ -174,6 +181,7 @@ def get_store(name: str) -> dict[str, Any] | None:
 
     Returns:
         Store configuration dictionary, or None if store doesn't exist.
+        Always returns docs_dirs as a list (converts legacy docs_dir if needed).
     """
     _validate_store_name(name)
     store_config_path = _get_store_dir(name) / STORE_CONFIG_FILE
@@ -184,22 +192,27 @@ def get_store(name: str) -> dict[str, Any] | None:
     try:
         with open(store_config_path, "r", encoding="utf-8") as f:
             result: dict[str, Any] = json.load(f)
-            return result
+
+        # Handle backwards compatibility: convert docs_dir to docs_dirs
+        if "docs_dirs" not in result and "docs_dir" in result:
+            result["docs_dirs"] = [result["docs_dir"]]
+
+        return result
     except (json.JSONDecodeError, OSError) as e:
         console.print(f"[yellow]Warning: Could not read store config: {e}[/yellow]")
         return None
 
 
 def create_store(
     name: str,
-    docs_dir: Path,
+    docs_dirs: list[Path],
     chat_model: str | None = None,
 ) -> dict[str, Any]:
     """Create a new document store.
 
     Args:
         name: Unique name for the store.
-        docs_dir: Path to the documents directory.
+        docs_dirs: List of paths to documents directories.
         chat_model: Optional chat model override for this store.
 
     Returns:
@@ -214,9 +227,12 @@ def create_store(
     if store_dir.exists():
         raise ValueError(f"Store '{name}' already exists.")
 
-    # Convert to absolute path if relative
-    if not docs_dir.is_absolute():
-        docs_dir = docs_dir.resolve()
+    # Convert to absolute paths if relative
+    resolved_dirs = []
+    for docs_dir in docs_dirs:
+        if not docs_dir.is_absolute():
+            docs_dir = docs_dir.resolve()
+        resolved_dirs.append(str(docs_dir))
 
     # Create store directory structure
     store_dir.mkdir(parents=True, exist_ok=True)
@@ -226,7 +242,7 @@ def create_store(
     # Create store configuration
     store_config = {
         "name": name,
-        "docs_dir": str(docs_dir),
+        "docs_dirs": resolved_dirs,
         "created": datetime.now(timezone.utc).isoformat(),
         "last_indexed": None,
         "chunk_count": 0,
@@ -266,12 +282,15 @@ def update_store(name: str, **kwargs: Any) -> dict[str, Any]:
 
     # Update fields
     for key, value in kwargs.items():
-        if key == "docs_dir" and value is not None:
-            # Ensure docs_dir is stored as absolute path string
-            path = Path(value)
-            if not path.is_absolute():
-                path = path.resolve()
-            store_config[key] = str(path)
+        if key == "docs_dirs" and value is not None:
+            # Ensure docs_dirs are stored as absolute path strings
+            resolved = []
+            for p in value:
+                path = Path(p) if isinstance(p, str) else p
+                if not path.is_absolute():
+                    path = path.resolve()
+                resolved.append(str(path))
+            store_config[key] = resolved
         else:
             store_config[key] = value
 

tests/unit/test_cli.py

@@ -451,7 +451,7 @@ def test_StoreNew_WithDocsArg_CreatesStore(self, mock_console: Any) -> None:
 
                     # Assert
                     assert result == 0
-                    mock_create.assert_called_once_with("my-store", Path("/path/to/docs"))
+                    mock_create.assert_called_once_with("my-store", [Path("/path/to/docs")])
 
     def test_StoreNew_WithExistingStore_ReturnsOne(self, mock_console: Any) -> None:
         """Test that store new returns 1 when store already exists."""
@@ -690,7 +690,7 @@ def test_StoreNew_WithoutDocsArg_PromptsAndSucceeds(self, mock_console: Any) ->
 
                         # Assert
                         assert result == 0
-                        mock_create.assert_called_once_with("my-store", Path("/path/to/docs"))
+                        mock_create.assert_called_once_with("my-store", [Path("/path/to/docs")])
 
     def test_StoreNew_WithEmptyDocsInput_ReturnsOne(self, mock_console: Any) -> None:
         """Test that store new returns 1 when user inputs empty docs path."""