Date: Tue, 17 Mar 2026 23:57:06 +0100
Subject: [PATCH 07/10] refactor(curriculum): audit-driven cleanup and
dossier-first explanation fallback
Curriculum YAML cleanup from product audit:
- Track 1: remove remedial concepts (number, fraction), merge redundant
concepts (sample-mean, deviation-from-mean, squared-error, l1-reg),
add one-hot-encoding and normalization-scaling to Module 9
- Track 2: swap Retrieval before Ranking (pipeline order), remove
feature-store (canonical home is Track 4), add embedding prerequisite
- Track 3: remove etl-vs-elt (redundant with Track 4), merge
alerting-thresholds and model-rollback into neighbors
- Track 5: rename batch-vs-real-time-serving to match Track 3 naming
Create 5 missing canonical concepts with minimal dossiers:
embedding, bias-variance-tradeoff, one-hot-encoding,
normalization-scaling, regularization
Fix concept explanation fallback to be dossier-first:
- Fetch dossier before attempting retrieval
- Dossier-only concepts produce real explanations (no dead-end)
- LLM fallback when neither dossier nor sources exist
- Three explicit provenance modes in API and UI:
dossier+retrieval, dossier_only, llm_fallback
---
backend/api/concepts.py | 1 +
.../jobs/seed_missing_curriculum_concepts.py | 134 ++++++++++++
.../services/learning/concept_explainer.py | 194 ++++++++++++------
curriculum/tracks/applied-systems.yaml | 2 +-
curriculum/tracks/ml-foundations.yaml | 54 ++---
curriculum/tracks/mlops.yaml | 10 +-
curriculum/tracks/practical-ml.yaml | 42 ++--
frontend/src/lib/api/types.ts | 1 +
frontend/src/views/ConceptDetailView.tsx | 14 +-
9 files changed, 317 insertions(+), 135 deletions(-)
create mode 100644 backend/jobs/seed_missing_curriculum_concepts.py
diff --git a/backend/api/concepts.py b/backend/api/concepts.py
index 7e77646..672437c 100644
--- a/backend/api/concepts.py
+++ b/backend/api/concepts.py
@@ -100,6 +100,7 @@ class ExplanationResponse(BaseModel):
source_titles: dict[str, str] = {}
dossier_grounded: bool = False
chunk_previews: list[ChunkPreviewResponse] = []
+ explanation_source: str = "dossier+retrieval"
class PracticeRequest(BaseModel):
diff --git a/backend/jobs/seed_missing_curriculum_concepts.py b/backend/jobs/seed_missing_curriculum_concepts.py
new file mode 100644
index 0000000..b22766d
--- /dev/null
+++ b/backend/jobs/seed_missing_curriculum_concepts.py
@@ -0,0 +1,134 @@
+"""Bootstrap missing canonical concepts required by curriculum tracks.
+
+Creates concept rows + minimal dossiers for concepts that exist in
+curriculum YAML but are missing from the canonical registry or lack dossiers.
+
+Usage:
+ python -m backend.jobs.seed_missing_curriculum_concepts
+ python -m backend.jobs.seed_missing_curriculum_concepts --dry-run
+"""
+
+import argparse
+import json
+import logging
+import sys
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+SRC = "curriculum:missing-concepts-v1"
+
+# (id, name, description, lens, level, intuition, formula, ml_usage)
+CONCEPTS = [
+ (
+ "embedding",
+ "Embedding",
+ "Learned dense vector representation that maps discrete items into continuous space.",
+ "machine_learning",
+ "university",
+ "Turn a word, user, or item into a list of numbers that captures its meaning.",
+ "embed(x) -> R^d",
+ "Retrieval, recommendations, semantic search, language models.",
+ ),
+ (
+ "bias-variance-tradeoff",
+ "Bias-Variance Tradeoff",
+ "Fundamental tension between underfitting (high bias) and overfitting (high variance) in model selection.",
+ "machine_learning",
+ "university",
+ "Simple models miss patterns; complex models memorize noise. The sweet spot is in between.",
+ "Error = Bias^2 + Variance + Noise",
+ "Model selection, regularization strength, ensemble methods.",
+ ),
+ (
+ "one-hot-encoding",
+ "One-Hot Encoding",
+ "Representation of a categorical variable as a binary vector with exactly one 1.",
+ "machine_learning",
+ "university",
+ "Turn a category like 'red' into [1, 0, 0] so a model can use it as a number.",
+ "[0, ..., 1, ..., 0]",
+ "Categorical feature preprocessing for linear models and trees.",
+ ),
+ (
+ "normalization-scaling",
+ "Normalization and Scaling",
+ "Transforming features to a common range or distribution before modeling.",
+ "machine_learning",
+ "university",
+ "Put all features on the same scale so no single feature dominates by magnitude.",
+ "z = (x - mean) / std or x' = (x - min) / (max - min)",
+ "Gradient-based training, distance-based models, convergence speed.",
+ ),
+ (
+ "regularization",
+ "Regularization",
+ "Technique that penalizes model complexity to reduce overfitting.",
+ "machine_learning",
+ "university",
+ "Add a cost for large weights so the model stays simple and generalizes better.",
+ "loss + lambda * penalty(w)",
+ "L1 sparsity, L2 shrinkage, dropout, early stopping.",
+ ),
+]
+
+
+def seed(dry_run: bool = False) -> bool:
+ """Seed missing concept rows + minimal dossiers."""
+ from backend.services.chat.history import get_db, init_db
+
+ init_db()
+ conn = get_db()
+
+ created = 0
+ for cid, name, desc, lens, level, intuition, formula, ml_usage in CONCEPTS:
+ # Concept row (INSERT OR REPLACE to handle existing-without-dossier cases)
+ conn.execute(
+ """INSERT OR REPLACE INTO concepts
+ (id, name, description, lens, level, source_count)
+ VALUES (?, ?, ?, ?, ?, 0)""",
+ (cid, name, desc, lens, level),
+ )
+
+ # Minimal dossier
+ formulas = (
+ [{"content": formula, "content_latex": formula,
+ "source_id": SRC, "source_title": None}]
+ if formula else []
+ )
+ insights = [{"content": ml_usage, "source_id": SRC}]
+ atom_count = 2 + (1 if formula else 0) + 1 # def + intuition + formula? + insight
+
+ conn.execute(
+ """INSERT OR REPLACE INTO concept_dossiers
+ (concept_id, formal_definition, intuitive_definition,
+ formulas_json, examples_json, prerequisite_claims_json,
+ key_insights_json, source_ids_json,
+ atom_count, source_count, built_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'))""",
+ (
+ cid, desc, intuition,
+ json.dumps(formulas), "[]", "[]",
+ json.dumps(insights), json.dumps([SRC]),
+ atom_count,
+ ),
+ )
+ created += 1
+ logger.info(" %s: concept + dossier", cid)
+
+ if dry_run:
+ logger.info("Dry run - %d concepts validated, rolling back", created)
+ conn.rollback()
+ else:
+ conn.commit()
+ logger.info("Seeded %d concepts + dossiers", created)
+
+ return True
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Bootstrap missing curriculum concepts")
+ parser.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args()
+ success = seed(dry_run=args.dry_run)
+ sys.exit(0 if success else 1)
diff --git a/backend/services/learning/concept_explainer.py b/backend/services/learning/concept_explainer.py
index cf6857c..bfb5f9d 100644
--- a/backend/services/learning/concept_explainer.py
+++ b/backend/services/learning/concept_explainer.py
@@ -41,6 +41,21 @@
- If the source material is thin, say so honestly
- Keep it concise - this is a focused explanation, not a textbook chapter"""
+LLM_FALLBACK_PROMPT = """You are an expert tutor explaining a concept to a motivated learner.
+
+CONCEPT: {concept_name}
+
+You have no source material for this concept. Generate a clear, accurate explanation from your training knowledge.
+
+Structure your response as:
+1. **Direct answer** (2-3 sentences explaining what this concept is and why it matters)
+2. **How it works** (bullet points covering the key mechanics or principles)
+3. **Example** (one concrete example)
+
+Rules:
+- Be accurate and concise
+- Keep it focused - this is a brief explanation, not a textbook chapter"""
+
PROFILED_EXPLAIN_PROMPT = """You are a tutor explaining a concept. Your teaching adapts to the learner's level and domain lens.
CONCEPT: {concept_name}
@@ -155,6 +170,8 @@ class ConceptExplanation(BaseModel):
source_titles: dict[str, str] = {}
dossier_grounded: bool = False
chunk_previews: list[ChunkPreview] = []
+ explanation_source: str = "dossier+retrieval"
+ # "dossier+retrieval" | "dossier_only" | "llm_fallback"
async def explain_concept(
@@ -167,67 +184,65 @@ async def explain_concept(
) -> ConceptExplanation:
"""Generate a grounded explanation for a concept.
- Retrieves relevant chunks from the concept's sources, then asks
- the LLM to synthesize an explanation using only that material.
+ Priority order:
+ 1. Dossier (>= 3 atoms) as primary source, retrieval as enrichment
+ 2. Dossier-only if retrieval finds nothing relevant
+ 3. LLM fallback if dossier is too thin AND no retrieval
"""
+ from backend.services.learning.dossier_builder import get_dossier
+
+ # Step 1: Fetch dossier first — it's always available for seeded concepts
+ dossier = get_dossier(concept_id)
+ has_dossier = dossier is not None and dossier.atom_count >= 2
+ has_rich_dossier = dossier is not None and dossier.atom_count >= 3
+
+ # Step 2: Attempt retrieval (enrichment, not required)
+ docs: list = []
+ scores: list[float] = []
from backend.services.retrieval import get_retriever
- # Retrieve chunks relevant to this concept
retriever = get_retriever("All")
- if not retriever:
- return ConceptExplanation(
- concept_id=concept_id,
- concept_name=concept_name,
- explanation=f"No source material available for '{concept_name}'. Ingest content that covers this concept first.",
- source_ids=[],
- chunk_count=0,
+ if retriever:
+ docs, timing = retriever.retrieve_with_timing(concept_name)
+ scores = timing.get("scores", [])
+
+ # Pad scores if missing (reranker unavailable)
+ if len(scores) < len(docs):
+ scores.extend([0.0] * (len(docs) - len(scores)))
+
+ # Filter by rerank relevance score — concept explanations need high relevance
+ # to avoid keyword-only matches (e.g. "Number" matching "this number is impressive")
+ CONCEPT_RELEVANCE_THRESHOLD = 0.90
+ if scores:
+ paired = [(d, s) for d, s in zip(docs, scores) if s >= CONCEPT_RELEVANCE_THRESHOLD]
+ if paired:
+ docs, scores = [list(t) for t in zip(*paired)]
+ else:
+ docs, scores = [], []
+
+ # Filter to chunks from sources that actually mention this concept
+ if source_ids and docs:
+ source_set = set(source_ids)
+ paired = [
+ (d, s) for d, s in zip(docs, scores)
+ if d.metadata.get("source_id") in source_set
+ or d.metadata.get("video_id") in source_set
+ ]
+ if paired:
+ docs, scores = [list(t) for t in zip(*paired)]
+
+ docs = docs[:max_chunks]
+
+ # Step 3: Decide explanation strategy
+ has_chunks = len(docs) > 0
+
+ if not has_dossier and not has_chunks:
+ # LLM fallback — no grounding material at all
+ return await _llm_fallback_explanation(
+ concept_id, concept_name, teaching_profile_id, teaching_lens,
)
- docs, timing = retriever.retrieve_with_timing(concept_name)
- scores: list[float] = timing.get("scores", [])
-
- # Pad scores if missing (reranker unavailable)
- if len(scores) < len(docs):
- scores.extend([0.0] * (len(docs) - len(scores)))
-
- # Filter by rerank relevance score — concept explanations need high relevance
- # to avoid keyword-only matches (e.g. "Number" matching "this number is impressive")
- # Concept explanations need high relevance — keyword-only matches (e.g.
- # "Number" matching "this number is impressive") score 70-82%, while
- # genuinely relevant chunks score 90%+. Use 0.90 to keep only real hits.
- CONCEPT_RELEVANCE_THRESHOLD = 0.90
- if scores:
- paired = [(d, s) for d, s in zip(docs, scores) if s >= CONCEPT_RELEVANCE_THRESHOLD]
- if paired:
- docs, scores = [list(t) for t in zip(*paired)]
- else:
- # All below threshold — fall back to dossier-only
- docs, scores = [], []
-
- # Filter to chunks from sources that actually mention this concept
- if source_ids and docs:
- source_set = set(source_ids)
- paired = [
- (d, s) for d, s in zip(docs, scores)
- if d.metadata.get("source_id") in source_set
- or d.metadata.get("video_id") in source_set
- ]
- # Fall back to all results if filtering removes everything
- if paired:
- docs, scores = [list(t) for t in zip(*paired)]
-
- docs = docs[:max_chunks]
-
- if not docs:
- return ConceptExplanation(
- concept_id=concept_id,
- concept_name=concept_name,
- explanation=f"No relevant source chunks found for '{concept_name}'. The concept exists in your learnings but the original source material may not contain enough detail.",
- source_ids=[],
- chunk_count=0,
- )
-
- # Build chunk context
+ # Build chunk context (may be empty — that's fine)
chunk_texts = []
seen_sources: set[str] = set()
source_titles: dict[str, str] = {}
@@ -239,7 +254,6 @@ async def explain_concept(
seen_sources.add(source_id)
source_titles[source_id] = title
raw = doc.page_content.strip()
- # Strip [Source: ...] prefix if baked into chunk text
if raw.startswith("[Source:"):
close = raw.find("]")
if close != -1:
@@ -250,15 +264,11 @@ async def explain_concept(
chunk_score = scores[idx] if idx < len(scores) else 0.0
chunk_previews.append(ChunkPreview(source_id=source_id, title=title, preview=preview, relevance_score=round(chunk_score, 3)))
- chunks_str = "\n\n---\n\n".join(chunk_texts)
+ chunks_str = "\n\n---\n\n".join(chunk_texts) if chunk_texts else "No supporting source chunks available."
- # Fetch dossier for structured substrate
- from backend.services.learning.dossier_builder import get_dossier
-
- dossier = get_dossier(concept_id)
dossier_section = _format_dossier_for_prompt(dossier)
- # Build prompt - use profiled version if teaching profile/lens provided
+ # Build prompt
if teaching_profile_id or teaching_lens:
profile = get_profile(teaching_profile_id or "university")
lens = get_lens(teaching_lens or "computer_science")
@@ -302,7 +312,6 @@ async def explain_concept(
settings = get_settings()
start = time.time()
- # Use Groq for fast explanation generation
llm = get_chat_llm(temperature=0.3, max_tokens=2000)
response = await ainvoke_with_observability(
llm,
@@ -320,8 +329,7 @@ async def explain_concept(
if teaching_profile_id or teaching_lens:
structured = _parse_structured_explanation(explanation_text)
- # Update concept description in DB only for default/canonical explanations.
- # Profiled explanations are variant renderings - not persisted as the canonical description.
+ # Update concept description in DB only for default/canonical explanations
if not (teaching_profile_id or teaching_lens):
from backend.services.learning.concept_store import _get_conn
@@ -332,7 +340,7 @@ async def explain_concept(
)
conn.commit()
- dossier_grounded = dossier is not None and dossier.atom_count > 0
+ explanation_source = "dossier+retrieval" if has_chunks else "dossier_only"
return ConceptExplanation(
concept_id=concept_id,
@@ -342,8 +350,60 @@ async def explain_concept(
source_ids=list(seen_sources),
chunk_count=len(docs),
source_titles=source_titles,
- dossier_grounded=dossier_grounded,
+ dossier_grounded=has_rich_dossier,
chunk_previews=chunk_previews,
+ explanation_source=explanation_source,
+ )
+
+
+async def _llm_fallback_explanation(
+ concept_id: str,
+ concept_name: str,
+ teaching_profile_id: str | None,
+ teaching_lens: str | None,
+) -> ConceptExplanation:
+ """Generate explanation purely from LLM knowledge when no dossier or sources exist."""
+ from backend.config import get_settings
+ from backend.services.llm.client import (
+ ainvoke_with_observability,
+ get_chat_llm,
+ record_llm_call,
+ )
+
+ logger.info("LLM fallback for concept '%s' (no dossier, no retrieval)", concept_id)
+
+ prompt = LLM_FALLBACK_PROMPT.format(concept_name=concept_name)
+
+ settings = get_settings()
+ start = time.time()
+
+ llm = get_chat_llm(temperature=0.3, max_tokens=2000)
+ response = await ainvoke_with_observability(
+ llm,
+ prompt,
+ operation="concept_explain_fallback",
+ provider="groq",
+ model=settings.chat_model,
+ )
+ record_llm_call(response, "groq", settings.chat_model, "concept_explain_fallback", start)
+
+ explanation_text = response.content.strip()
+
+ structured = None
+ if teaching_profile_id or teaching_lens:
+ structured = _parse_structured_explanation(explanation_text)
+
+ return ConceptExplanation(
+ concept_id=concept_id,
+ concept_name=concept_name,
+ explanation=explanation_text,
+ structured=structured,
+ source_ids=[],
+ chunk_count=0,
+ source_titles={},
+ dossier_grounded=False,
+ chunk_previews=[],
+ explanation_source="llm_fallback",
)
diff --git a/curriculum/tracks/applied-systems.yaml b/curriculum/tracks/applied-systems.yaml
index d128cea..82d1726 100644
--- a/curriculum/tracks/applied-systems.yaml
+++ b/curriculum/tracks/applied-systems.yaml
@@ -88,7 +88,7 @@ modules:
concepts:
- concept_id: latency-throughput-tradeoff
sort_order: 1
- - concept_id: batch-vs-real-time-serving
+ - concept_id: batch-vs-realtime-inference
sort_order: 2
- concept_id: model-complexity-tradeoff
sort_order: 3
diff --git a/curriculum/tracks/ml-foundations.yaml b/curriculum/tracks/ml-foundations.yaml
index f3d4095..7f333f5 100644
--- a/curriculum/tracks/ml-foundations.yaml
+++ b/curriculum/tracks/ml-foundations.yaml
@@ -21,24 +21,18 @@ modules:
sort_order: 1
color: "#55cdff"
concepts:
- - concept_id: number
- sort_order: 1
- - concept_id: fraction
- sort_order: 2
- concept_id: ratio
- sort_order: 3
+ sort_order: 1
- concept_id: variable
- sort_order: 4
+ sort_order: 2
- concept_id: algebraic-expression
- sort_order: 5
+ sort_order: 3
- concept_id: function
- sort_order: 6
+ sort_order: 4
- concept_id: exponent
- sort_order: 7
+ sort_order: 5
- concept_id: logarithm
- sort_order: 8
- - concept_id: square-root
- sort_order: 9
+ sort_order: 6
# ── Module 2: Describing Data ──────────────────────────────────────────
- id: describing-data
@@ -52,20 +46,16 @@ modules:
concepts:
- concept_id: mean
sort_order: 1
- - concept_id: sample-mean
- sort_order: 2
- - concept_id: deviation-from-mean
- sort_order: 3
- concept_id: variance
- sort_order: 4
+ sort_order: 2
- concept_id: standard-deviation
- sort_order: 5
+ sort_order: 3
- concept_id: covariance
- sort_order: 6
+ sort_order: 4
- concept_id: correlation
- sort_order: 7
+ sort_order: 5
- concept_id: z-score
- sort_order: 8
+ sort_order: 6
# ── Module 3: Probability and Bayes ────────────────────────────────────
- id: probability-and-bayes
@@ -119,10 +109,8 @@ modules:
concepts:
- concept_id: feature
sort_order: 1
- - concept_id: linear-model
- sort_order: 2
- concept_id: linear-regression
- sort_order: 3
+ sort_order: 2
# ── Module 6: Probabilities for Classification ─────────────────────────
- id: probabilities-for-classification
@@ -153,14 +141,12 @@ modules:
sort_order: 7
color: "#55cdff"
concepts:
- - concept_id: squared-error
- sort_order: 1
- concept_id: mse-loss
- sort_order: 2
+ sort_order: 1
- concept_id: cross-entropy-loss
- sort_order: 3
+ sort_order: 2
- concept_id: gradient-descent
- sort_order: 4
+ sort_order: 3
# ── Module 8: Generalization and Evaluation ────────────────────────────
- id: generalization-and-evaluation
@@ -174,11 +160,11 @@ modules:
concepts:
- concept_id: overfitting
sort_order: 1
- - concept_id: cross-validation
+ - concept_id: bias-variance-tradeoff
sort_order: 2
- - concept_id: l1-regularization
+ - concept_id: cross-validation
sort_order: 3
- - concept_id: l2-regularization
+ - concept_id: regularization
sort_order: 4
# ── Module 9: Features and Practical Modeling ──────────────────────────
@@ -193,3 +179,7 @@ modules:
concepts:
- concept_id: feature-engineering
sort_order: 1
+ - concept_id: one-hot-encoding
+ sort_order: 2
+ - concept_id: normalization-scaling
+ sort_order: 3
diff --git a/curriculum/tracks/mlops.yaml b/curriculum/tracks/mlops.yaml
index b265fb9..c95a970 100644
--- a/curriculum/tracks/mlops.yaml
+++ b/curriculum/tracks/mlops.yaml
@@ -29,8 +29,6 @@ modules:
sort_order: 3
- concept_id: data-versioning
sort_order: 4
- - concept_id: etl-vs-elt
- sort_order: 5
# ── Module 2: Training Systems ───────────────────────────────────────
- id: training-systems
@@ -90,8 +88,6 @@ modules:
sort_order: 2
- concept_id: model-performance-monitoring
sort_order: 3
- - concept_id: alerting-thresholds
- sort_order: 4
# ── Module 5: Lifecycle ──────────────────────────────────────────────
- id: lifecycle
@@ -107,9 +103,7 @@ modules:
sort_order: 1
- concept_id: canary-rollout
sort_order: 2
- - concept_id: model-rollback
- sort_order: 3
- concept_id: model-governance
- sort_order: 4
+ sort_order: 3
- concept_id: technical-debt-ml
- sort_order: 5
+ sort_order: 4
diff --git a/curriculum/tracks/practical-ml.yaml b/curriculum/tracks/practical-ml.yaml
index 6df922c..405f844 100644
--- a/curriculum/tracks/practical-ml.yaml
+++ b/curriculum/tracks/practical-ml.yaml
@@ -32,44 +32,44 @@ modules:
- concept_id: cold-start-problem
sort_order: 5
- # ── Module 2: Ranking Systems ─────────────────────────────────────────
- - id: ranking-systems
- title: Ranking Systems
+ # ── Module 2: Retrieval Systems ───────────────────────────────────────
+ - id: retrieval-systems
+ title: Retrieval Systems
objective: >
- Learn how search engines and feeds order results using pointwise,
- pairwise, and listwise learning-to-rank approaches.
- estimated_time_minutes: 60
+ Build intuition for approximate nearest neighbor search, two-tower
+ models, and the retrieval-then-rank pipeline used in modern systems.
+ estimated_time_minutes: 45
sort_order: 2
color: "#4ade80"
concepts:
- - concept_id: pointwise-ranking
+ - concept_id: embedding
sort_order: 1
- - concept_id: pairwise-ranking
+ - concept_id: embedding-similarity
sort_order: 2
- - concept_id: listwise-ranking
+ - concept_id: approximate-nearest-neighbor
sort_order: 3
- - concept_id: ndcg
+ - concept_id: two-tower-model
sort_order: 4
- - concept_id: feature-store
+ - concept_id: retrieval-then-rank
sort_order: 5
- # ── Module 3: Retrieval Systems ───────────────────────────────────────
- - id: retrieval-systems
- title: Retrieval Systems
+ # ── Module 3: Ranking Systems ─────────────────────────────────────────
+ - id: ranking-systems
+ title: Ranking Systems
objective: >
- Build intuition for approximate nearest neighbor search, two-tower
- models, and the retrieval-then-rank pipeline used in modern systems.
- estimated_time_minutes: 45
+ Learn how search engines and feeds order results using pointwise,
+ pairwise, and listwise learning-to-rank approaches.
+ estimated_time_minutes: 60
sort_order: 3
color: "#ffc47c"
concepts:
- - concept_id: embedding-similarity
+ - concept_id: pointwise-ranking
sort_order: 1
- - concept_id: approximate-nearest-neighbor
+ - concept_id: pairwise-ranking
sort_order: 2
- - concept_id: two-tower-model
+ - concept_id: listwise-ranking
sort_order: 3
- - concept_id: retrieval-then-rank
+ - concept_id: ndcg
sort_order: 4
# ── Module 4: Feedback Loops ──────────────────────────────────────────
diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts
index 2b9e0e7..7ae6c28 100644
--- a/frontend/src/lib/api/types.ts
+++ b/frontend/src/lib/api/types.ts
@@ -1510,6 +1510,7 @@ export interface ConceptExplanation {
source_titles: Record
;
dossier_grounded: boolean;
chunk_previews: ChunkPreview[];
+ explanation_source: "dossier+retrieval" | "dossier_only" | "llm_fallback";
}
export interface PracticeItem {
diff --git a/frontend/src/views/ConceptDetailView.tsx b/frontend/src/views/ConceptDetailView.tsx
index c96c6eb..a2830b5 100644
--- a/frontend/src/views/ConceptDetailView.tsx
+++ b/frontend/src/views/ConceptDetailView.tsx
@@ -502,7 +502,7 @@ function ExplanationResult({ data }: { data: ConceptExplanation }) {
const sourceTitles = data.source_titles ?? {};
const chunkPreviews = data.chunk_previews ?? [];
- const isDossierOnly = data.dossier_grounded && data.chunk_count === 0;
+ const explSource = data.explanation_source ?? "dossier+retrieval";
return (
- {isDossierOnly ? (
+ {explSource === "llm_fallback" ? (
- Based on internal concept dossier
+ AI-generated (no sources)
+
+ ) : explSource === "dossier_only" ? (
+
+ Based on concept dossier - no source material available yet
) : (
<>
- Based on {data.chunk_count} chunk
- {data.chunk_count !== 1 ? "s" : ""} from {data.source_ids.length}{" "}
- source
+ Based on {data.source_ids.length} source
{data.source_ids.length !== 1 ? "s" : ""}
{data.dossier_grounded ? " + concept dossier" : ""}
From 6507c5854d108aeac7b78dcac7b00df05992e8e7 Mon Sep 17 00:00:00 2001
From: scaleborg <218523607+scaleborg@users.noreply.github.com>
Date: Wed, 18 Mar 2026 00:00:08 +0100
Subject: [PATCH 08/10] fix(tests): update curriculum test fixture schema to
match production
Test fixture was using old schema (sort_order column, no track_type/
category/color). Updated to match current production tables. Also
updated stale YAML assertions for ml-foundations (9 modules, not 3).
---
tests/test_curriculum.py | 37 ++++++++++++++++++++++++++-----------
1 file changed, 26 insertions(+), 11 deletions(-)
diff --git a/tests/test_curriculum.py b/tests/test_curriculum.py
index 4fdc52c..b146287 100644
--- a/tests/test_curriculum.py
+++ b/tests/test_curriculum.py
@@ -64,8 +64,11 @@ def db():
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
difficulty TEXT NOT NULL DEFAULT 'intermediate',
- sort_order INTEGER NOT NULL DEFAULT 0,
+ track_type TEXT NOT NULL DEFAULT 'concept',
is_published INTEGER NOT NULL DEFAULT 1,
+ category TEXT NOT NULL DEFAULT 'core',
+ category_order INTEGER NOT NULL DEFAULT 1,
+ track_order INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
@@ -77,7 +80,7 @@ def db():
objective TEXT NOT NULL DEFAULT '',
estimated_time_minutes INTEGER NOT NULL DEFAULT 30,
sort_order INTEGER NOT NULL DEFAULT 0,
- is_published INTEGER NOT NULL DEFAULT 1,
+ color TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (id, track_id),
@@ -96,6 +99,21 @@ def db():
FOREIGN KEY (concept_id) REFERENCES concepts(id) ON DELETE CASCADE
);
+ CREATE TABLE track_resources (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ track_id TEXT NOT NULL,
+ module_id TEXT NOT NULL,
+ name TEXT NOT NULL,
+ url TEXT,
+ description TEXT,
+ detail TEXT,
+ resource_type TEXT NOT NULL DEFAULT 'link',
+ sort_order INTEGER NOT NULL DEFAULT 0,
+ metadata_json TEXT,
+ created_at TEXT NOT NULL DEFAULT (datetime('now')),
+ FOREIGN KEY (track_id) REFERENCES curriculum_tracks(id) ON DELETE CASCADE
+ );
+
CREATE TABLE concept_progress (
user_id TEXT NOT NULL DEFAULT 'default',
concept_id TEXT NOT NULL,
@@ -464,12 +482,9 @@ def test_load_ml_foundations_yaml():
track = load_track_file(path)
assert track.id == "ml-foundations"
- assert len(track.modules) == 3
- assert track.modules[0].id == "vector-similarity"
- assert len(track.modules[0].concepts) == 4
- # Normalized IDs
- assert track.modules[0].concepts[0].concept_id == "dense-vectors"
- assert track.modules[0].concepts[3].concept_id == "vector-embeddings"
- # Attention module (narrowed from transformers)
- assert track.modules[2].id == "attention"
- assert len(track.modules[2].concepts) == 2
+ assert len(track.modules) == 9
+ assert track.modules[0].id == "numbers-formulas-functions"
+ assert track.modules[0].concepts[0].concept_id == "ratio"
+ # Module 8: generalization includes bias-variance-tradeoff
+ assert track.modules[7].id == "generalization-and-evaluation"
+ assert track.modules[7].concepts[1].concept_id == "bias-variance-tradeoff"
From 58569860cea88fdf007173eff14ef8073d36a9c2 Mon Sep 17 00:00:00 2001
From: scaleborg <218523607+scaleborg@users.noreply.github.com>
Date: Wed, 18 Mar 2026 00:03:19 +0100
Subject: [PATCH 09/10] fix(lint): remove unused import and rename ambiguous
variable in merge_concepts
Fixes ruff F401 (unused `field` import) and E741 (ambiguous `l` variable).
---
backend/jobs/merge_concepts.py | 54 +++++++++++++++++-----------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/backend/jobs/merge_concepts.py b/backend/jobs/merge_concepts.py
index 5003821..7e434fb 100644
--- a/backend/jobs/merge_concepts.py
+++ b/backend/jobs/merge_concepts.py
@@ -11,7 +11,7 @@
import argparse
import sqlite3
import sys
-from dataclasses import dataclass, field
+from dataclasses import dataclass
from pathlib import Path
DB_PATH = Path("data/samaritan.db")
@@ -49,21 +49,21 @@ class MergeStats:
def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> MergeStats:
"""Merge loser into winner. Returns stats of what was (or would be) done."""
stats = MergeStats()
- w, l = pair.winner, pair.loser
+ w, loser_id = pair.winner, pair.loser
# Verify both exist
winner_row = conn.execute("SELECT id, source_count FROM concepts WHERE id = ?", (w,)).fetchone()
- loser_row = conn.execute("SELECT id, source_count FROM concepts WHERE id = ?", (l,)).fetchone()
+ loser_row = conn.execute("SELECT id, source_count FROM concepts WHERE id = ?", (loser_id,)).fetchone()
if not winner_row:
print(f" ERROR: winner {w!r} not found in concepts table")
return stats
if not loser_row:
- print(f" WARNING: loser {l!r} not found — already merged?")
+ print(f" WARNING: loser {loser_id!r} not found — already merged?")
return stats
# 1. Absorb aliases: add loser's aliases pointing to winner (skip dupes)
loser_aliases = conn.execute(
- "SELECT alias FROM concept_aliases WHERE concept_id = ?", (l,)
+ "SELECT alias FROM concept_aliases WHERE concept_id = ?", (loser_id,)
).fetchall()
winner_aliases = {
row[0]
@@ -78,7 +78,7 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
# Delete from loser first (PK is alias+concept_id)
conn.execute(
"DELETE FROM concept_aliases WHERE alias = ? AND concept_id = ?",
- (alias, l),
+ (alias, loser_id),
)
conn.execute(
"INSERT OR IGNORE INTO concept_aliases (alias, concept_id) VALUES (?, ?)",
@@ -89,26 +89,26 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
if execute:
conn.execute(
"DELETE FROM concept_aliases WHERE alias = ? AND concept_id = ?",
- (alias, l),
+ (alias, loser_id),
)
# 2. Move concept_atoms (concept_id)
atom_count = conn.execute(
- "SELECT COUNT(*) FROM concept_atoms WHERE concept_id = ?", (l,)
+ "SELECT COUNT(*) FROM concept_atoms WHERE concept_id = ?", (loser_id,)
).fetchone()[0]
stats.atoms_moved = atom_count
if execute and atom_count:
- conn.execute("UPDATE concept_atoms SET concept_id = ? WHERE concept_id = ?", (w, l))
+ conn.execute("UPDATE concept_atoms SET concept_id = ? WHERE concept_id = ?", (w, loser_id))
# 3. Move concept_atoms (target_concept_id)
target_count = conn.execute(
- "SELECT COUNT(*) FROM concept_atoms WHERE target_concept_id = ?", (l,)
+ "SELECT COUNT(*) FROM concept_atoms WHERE target_concept_id = ?", (loser_id,)
).fetchone()[0]
stats.atom_targets_moved = target_count
if execute and target_count:
conn.execute(
"UPDATE concept_atoms SET target_concept_id = ? WHERE target_concept_id = ?",
- (w, l),
+ (w, loser_id),
)
# 4. Handle concept_dossiers (1:1 — winner takes priority)
@@ -116,7 +116,7 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
"SELECT concept_id FROM concept_dossiers WHERE concept_id = ?", (w,)
).fetchone()
loser_dossier = conn.execute(
- "SELECT concept_id FROM concept_dossiers WHERE concept_id = ?", (l,)
+ "SELECT concept_id FROM concept_dossiers WHERE concept_id = ?", (loser_id,)
).fetchone()
if loser_dossier:
if winner_dossier:
@@ -127,17 +127,17 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
if execute:
conn.execute(
"UPDATE concept_dossiers SET concept_id = ? WHERE concept_id = ?",
- (w, l),
+ (w, loser_id),
)
stats.dossiers_deleted = 1 if winner_dossier else 0
if execute and winner_dossier:
- conn.execute("DELETE FROM concept_dossiers WHERE concept_id = ?", (l,))
+ conn.execute("DELETE FROM concept_dossiers WHERE concept_id = ?", (loser_id,))
# 5. Move concept_prerequisites (both directions, skip dupes)
for col in ("concept_id", "prerequisite_id"):
other_col = "prerequisite_id" if col == "concept_id" else "concept_id"
rows = conn.execute(
- f"SELECT {other_col} FROM concept_prerequisites WHERE {col} = ?", (l,)
+ f"SELECT {other_col} FROM concept_prerequisites WHERE {col} = ?", (loser_id,)
).fetchall()
for (other_id,) in rows:
# Check if winner already has this edge
@@ -156,26 +156,26 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
if execute:
conn.execute(
f"UPDATE concept_prerequisites SET {col} = ? WHERE {col} = ? AND {other_col} = ?",
- (w, l, other_id),
+ (w, loser_id, other_id),
)
else:
if execute:
conn.execute(
f"DELETE FROM concept_prerequisites WHERE {col} = ? AND {other_col} = ?",
- (l, other_id),
+ (loser_id, other_id),
)
# 6. Move practice_items
practice_count = conn.execute(
- "SELECT COUNT(*) FROM practice_items WHERE concept_id = ?", (l,)
+ "SELECT COUNT(*) FROM practice_items WHERE concept_id = ?", (loser_id,)
).fetchone()[0]
stats.practice_moved = practice_count
if execute and practice_count:
- conn.execute("UPDATE practice_items SET concept_id = ? WHERE concept_id = ?", (w, l))
+ conn.execute("UPDATE practice_items SET concept_id = ? WHERE concept_id = ?", (w, loser_id))
# 7. Move concept_progress (skip if winner already has entry for same user)
progress_rows = conn.execute(
- "SELECT user_id FROM concept_progress WHERE concept_id = ?", (l,)
+ "SELECT user_id FROM concept_progress WHERE concept_id = ?", (loser_id,)
).fetchall()
for (user_id,) in progress_rows:
exists = conn.execute(
@@ -187,19 +187,19 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
if execute:
conn.execute(
"UPDATE concept_progress SET concept_id = ? WHERE user_id = ? AND concept_id = ?",
- (w, user_id, l),
+ (w, user_id, loser_id),
)
else:
if execute:
conn.execute(
"DELETE FROM concept_progress WHERE user_id = ? AND concept_id = ?",
- (user_id, l),
+ (user_id, loser_id),
)
# 8. Move curriculum_module_concepts (skip dupes)
curriculum_rows = conn.execute(
"SELECT module_id, track_id FROM curriculum_module_concepts WHERE concept_id = ?",
- (l,),
+ (loser_id,),
).fetchall()
for module_id, track_id in curriculum_rows:
exists = conn.execute(
@@ -211,13 +211,13 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
if execute:
conn.execute(
"UPDATE curriculum_module_concepts SET concept_id = ? WHERE module_id = ? AND track_id = ? AND concept_id = ?",
- (w, module_id, track_id, l),
+ (w, module_id, track_id, loser_id),
)
else:
if execute:
conn.execute(
"DELETE FROM curriculum_module_concepts WHERE module_id = ? AND track_id = ? AND concept_id = ?",
- (module_id, track_id, l),
+ (module_id, track_id, loser_id),
)
# 9. Update winner source_count (additive from loser, avoid double-count)
@@ -232,8 +232,8 @@ def merge_one(conn: sqlite3.Connection, pair: MergePair, execute: bool) -> Merge
# 10. Delete remaining loser aliases (cleanup) then loser concept
if execute:
- conn.execute("DELETE FROM concept_aliases WHERE concept_id = ?", (l,))
- conn.execute("DELETE FROM concepts WHERE id = ?", (l,))
+ conn.execute("DELETE FROM concept_aliases WHERE concept_id = ?", (loser_id,))
+ conn.execute("DELETE FROM concepts WHERE id = ?", (loser_id,))
return stats
From 1e88312a004a47e3d4480cb9a055121436414046 Mon Sep 17 00:00:00 2001
From: scaleborg <218523607+scaleborg@users.noreply.github.com>
Date: Wed, 18 Mar 2026 10:33:20 +0100
Subject: [PATCH 10/10] chore: trigger CI with updated Groq API key