From 2fd9f51d2feec3a951b984176fc2ba78e6d2d7f1 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 7 Apr 2026 17:13:02 -0400 Subject: [PATCH 01/12] =?UTF-8?q?Add=20retrieval-side=20abstractions=20to?= =?UTF-8?q?=20MEDI=20=E2=80=94=20mirror=20ingestion=20pattern?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Abstractions (Microsoft.Extensions.DataIngestion.Abstractions): - RetrievalQuery: query data type with variants + metadata - RetrievalChunk: single chunk with content, score, record data - RetrievalResults: result collection with pipeline metadata - RetrievalQueryProcessor: abstract base for pre-search processing - RetrievalResultProcessor: abstract base for post-search processing - ISearchReranker: interface for re-ranking strategies Implementation (Microsoft.Extensions.DataIngestion): - RetrievalPipeline: orchestrator (query processors → vector search → result processors) - RetrievalPipelineOptions: ActivitySource configuration - Source-generated log methods for pipeline tracing Mirrors IngestionPipeline design: empty processor list = raw vector search. Builds clean: 0 warnings, 0 errors across all target frameworks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- global.json | 3 +- .../ISearchReranker.cs | 28 +++ .../RetrievalChunk.cs | 42 ++++ .../RetrievalQuery.cs | 42 ++++ .../RetrievalQueryProcessor.cs | 26 +++ .../RetrievalResultProcessor.cs | 27 +++ .../RetrievalResults.cs | 26 +++ .../Microsoft.Extensions.DataIngestion/Log.cs | 9 + .../RetrievalPipeline.cs | 183 ++++++++++++++++++ .../RetrievalPipelineOptions.cs | 22 +++ 10 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs diff --git a/global.json b/global.json index 4d9009f7346..f77fee427fe 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,7 @@ { "sdk": { - "version": "10.0.105" + "version": "10.0.105", + "rollForward": "latestMajor" }, "tools": { "dotnet": "10.0.105", diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs new file mode 100644 index 00000000000..96a12eb34e0 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Defines a re-ranking strategy for retrieval results. +/// +/// +/// Re-rankers score and reorder retrieval chunks based on relevance to the query. +/// Implementations may use LLM-based scoring, cross-encoder models (e.g., ONNX), +/// or other ranking strategies. +/// +public interface ISearchReranker +{ + /// + /// Re-ranks the provided chunks based on their relevance to the query. + /// + /// The search query. + /// The chunks to re-rank. + /// The token to monitor for cancellation requests. + /// The re-ranked chunks, ordered by relevance (highest first). + Task> RerankAsync(string query, IReadOnlyList chunks, CancellationToken cancellationToken = default); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs new file mode 100644 index 00000000000..d7aabdbb1aa --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Represents a single chunk returned from retrieval. +/// +public sealed class RetrievalChunk +{ + /// + /// Initializes a new instance of the class. + /// + /// The text content of the chunk. + /// The relevance score from vector search. + public RetrievalChunk(string content, double score) + { + Content = content; + Score = score; + } + + /// + /// Gets the text content of this chunk. + /// + public string Content { get; } + + /// + /// Gets or sets the relevance score. + /// + public double Score { get; set; } + + /// + /// Gets the underlying record data as key-value pairs. + /// + /// + /// Contains the full record fields from the vector store, enabling + /// downstream consumers to reconstruct strongly-typed records. + /// + public IDictionary Record { get; } = new Dictionary(); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs new file mode 100644 index 00000000000..e50f3f0e691 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Represents a retrieval query with optional expanded variants and metadata. +/// +public sealed class RetrievalQuery +{ + /// + /// Initializes a new instance of the class. + /// + /// The original query text. + public RetrievalQuery(string text) + { + Text = text; + Variants = [text]; + } + + /// + /// Gets the original query text. + /// + public string Text { get; } + + /// + /// Gets or sets the query variants to search with. + /// + /// + /// Pre-query processors may expand a single query into multiple variants + /// (e.g., multi-query expansion, HyDE). Each variant is searched independently + /// and results are merged using Reciprocal Rank Fusion. + /// + public IList Variants { get; set; } + + /// + /// Gets the metadata associated with this query. + /// + public IDictionary Metadata { get; } = new Dictionary(); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs new file mode 100644 index 00000000000..38689a97ed7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Processes a retrieval query before vector search is performed. +/// +/// +/// Pre-search processors transform or expand a +/// before it is sent to the vector store. Examples include multi-query expansion, +/// HyDE (Hypothetical Document Embeddings), and adaptive routing. +/// +public abstract class RetrievalQueryProcessor +{ + /// + /// Processes the query asynchronously before vector search. + /// + /// The retrieval query to process. + /// The token to monitor for cancellation requests. + /// The processed query. + public abstract Task ProcessQueryAsync(RetrievalQuery query, CancellationToken cancellationToken = default); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs new file mode 100644 index 00000000000..f63639028a0 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Processes retrieval results after vector search is performed. +/// +/// +/// Post-search processors transform or filter +/// after they are returned from the vector store. Examples include re-ranking, +/// CRAG (Corrective RAG) quality validation, and deduplication. +/// +public abstract class RetrievalResultProcessor +{ + /// + /// Processes the results asynchronously after vector search. + /// + /// The retrieval results to process. + /// The original query (for context during processing). + /// The token to monitor for cancellation requests. + /// The processed results. + public abstract Task ProcessResultsAsync(RetrievalResults results, RetrievalQuery query, CancellationToken cancellationToken = default); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs new file mode 100644 index 00000000000..61f45d1fdad --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Represents the results of a retrieval operation. +/// +public sealed class RetrievalResults +{ + /// + /// Gets or sets the retrieved chunks, ordered by relevance. + /// + public IList Chunks { get; set; } = []; + + /// + /// Gets the metadata from the retrieval pipeline. + /// + /// + /// Pipeline processors may add metadata such as CRAG quality scores, + /// reranking diagnostics, or query expansion details. + /// + public IDictionary Metadata { get; } = new Dictionary(); +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs index 58732e8ead7..8fcb78f1b77 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs @@ -37,5 +37,14 @@ internal static partial class Log [LoggerMessage(8, LogLevel.Error, "Unexpected enricher failure.")] internal static partial void UnexpectedEnricherFailure(this ILogger logger, Exception exception); + + [LoggerMessage(9, LogLevel.Debug, "Running query processor: {processor}.")] + internal static partial void RunningQueryProcessor(this ILogger logger, string processor); + + [LoggerMessage(10, LogLevel.Debug, "Searching variant: {variant}.")] + internal static partial void SearchingVariant(this ILogger logger, string variant); + + [LoggerMessage(11, LogLevel.Debug, "Running result processor: {processor}.")] + internal static partial void RunningResultProcessor(this ILogger logger, string processor); } } diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs new file mode 100644 index 00000000000..88bf29e7a9b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.VectorData; +using Microsoft.Shared.Diagnostics; +using static Microsoft.Extensions.DataIngestion.DiagnosticsConstants; + +namespace Microsoft.Extensions.DataIngestion; + +#pragma warning disable IDE0058 // Expression value is never used +#pragma warning disable IDE0063 // Use simple 'using' statement + +/// +/// Represents a pipeline for retrieving data from a vector store with pre- and post-processing. +/// +/// +/// Mirrors design. +/// Flow: query → → vector search → → results. +/// With an empty processor list, this behaves identically to a raw +/// call. +/// +public sealed class RetrievalPipeline : IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly ILogger? _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The options for the retrieval pipeline. + /// The logger factory for creating loggers. + public RetrievalPipeline( + RetrievalPipelineOptions? options = null, + ILoggerFactory? loggerFactory = null) + { + _activitySource = new((options ?? new()).ActivitySourceName); + _logger = loggerFactory?.CreateLogger(); + } + + /// + public void Dispose() + { + _activitySource.Dispose(); + } + + /// + /// Gets the pre-search query processors (e.g., query expansion, HyDE). + /// + public IList QueryProcessors { get; } = []; + + /// + /// Gets the post-search result processors (e.g., re-ranking, CRAG). + /// + public IList ResultProcessors { get; } = []; + + /// + /// Executes the retrieval pipeline: query processing → vector search → result processing. + /// + /// The vector store key type. + /// The vector store record type. + /// The vector store collection to search. + /// The user query. + /// Maximum results to retrieve per search variant. + /// Extracts text content from a record for result processing. + /// The token to monitor for cancellation requests. + /// The retrieval results. + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2075", + Justification = "Record properties are accessed for diagnostic metadata population only.")] + public async Task RetrieveAsync( + VectorStoreCollection collection, + string query, + int topK = 5, + Func? contentSelector = null, + CancellationToken cancellationToken = default) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(collection); + Throw.IfNullOrEmpty(query); + + using (Activity? rootActivity = _activitySource.StartActivity("RetrievalPipeline.Retrieve")) + { + rootActivity?.SetTag("rag.query", query) + .SetTag("rag.topK", topK); + + // Phase 1: Pre-query processing + var retrievalQuery = new RetrievalQuery(query); + + foreach (var processor in QueryProcessors) + { + _logger?.RunningQueryProcessor(processor.GetType().Name); + retrievalQuery = await processor.ProcessQueryAsync(retrievalQuery, cancellationToken).ConfigureAwait(false); + } + + rootActivity?.SetTag("rag.query.variants", retrievalQuery.Variants.Count); + + // Phase 2: Vector search (one search per variant, merge with RRF) + var allChunks = new List(); + + foreach (string variant in retrievalQuery.Variants) + { + _logger?.SearchingVariant(variant.Length > 80 ? variant[..80] : variant); + var searchResults = collection.SearchAsync(variant, top: topK, cancellationToken: cancellationToken); + + await foreach (var result in searchResults.ConfigureAwait(false)) + { + string content = contentSelector is not null && result.Record is not null + ? contentSelector(result.Record) + : result.Record?.ToString() ?? string.Empty; + + var chunk = new RetrievalChunk(content, result.Score ?? 0.0); + + // Populate record dictionary for downstream reconstruction + if (result.Record is not null) + { + foreach (var prop in result.Record.GetType().GetProperties()) + { + chunk.Record[prop.Name] = prop.GetValue(result.Record); + } + } + + allChunks.Add(chunk); + } + } + + // Deduplicate by content if multiple variants returned overlapping results + if (retrievalQuery.Variants.Count > 1) + { + allChunks = DeduplicateWithRrf(allChunks); + } + + var results = new RetrievalResults { Chunks = allChunks }; + rootActivity?.SetTag("rag.results.count", results.Chunks.Count); + + // Phase 3: Post-search result processing + foreach (var processor in ResultProcessors) + { + _logger?.RunningResultProcessor(processor.GetType().Name); + results = await processor.ProcessResultsAsync(results, retrievalQuery, cancellationToken).ConfigureAwait(false); + } + + rootActivity?.SetTag("rag.results.final_count", results.Chunks.Count); + return results; + } + } + + /// + /// Deduplicates chunks using Reciprocal Rank Fusion across multiple query variants. + /// + private static List DeduplicateWithRrf(List chunks) + { + const int RrfK = 60; + + var grouped = chunks + .GroupBy(c => c.Content) + .Select(g => new + { + Chunk = g.First(), + RrfScore = g.Sum(c => + { + int rank = chunks.IndexOf(c) + 1; + return 1.0 / (RrfK + rank); + }) + }) + .OrderByDescending(x => x.RrfScore) + .ToList(); + + foreach (var item in grouped) + { + item.Chunk.Score = item.RrfScore; + } + + return grouped.Select(x => x.Chunk).ToList(); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs new file mode 100644 index 00000000000..70f031f2f8d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Options for configuring the retrieval pipeline. +/// +public sealed class RetrievalPipelineOptions +{ + /// + /// Gets or sets the name of the used for diagnostics. + /// + public string ActivitySourceName + { + get; + set => field = Throw.IfNullOrEmpty(value); + } = DiagnosticsConstants.ActivitySourceName; +} From f6637759491ead0f6d156b4b8c0e606bcc560bd1 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 8 Apr 2026 10:08:04 -0400 Subject: [PATCH 02/12] Rename ProcessQueryAsync/ProcessResultsAsync to ProcessAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align retrieval processor method names with ingestion-side convention. IngestionChunkProcessor and IngestionDocumentProcessor both use ProcessAsync — the retrieval abstractions should follow suit. - RetrievalQueryProcessor.ProcessQueryAsync → ProcessAsync - RetrievalResultProcessor.ProcessResultsAsync → ProcessAsync - RetrievalPipeline: update calls to use new names Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../RetrievalQueryProcessor.cs | 2 +- .../RetrievalResultProcessor.cs | 2 +- .../RetrievalPipeline.cs | 4 +- .../.template.config/dotnetcli.host.json | 7 + .../.template.config/ide.host.json | 6 + .../.template.config/ide/icon.ico | Bin 0 -> 38045 bytes .../.template.config/template.json | 134 ++++++++++++++++++ .../AgentsWebAPI-CSharp.csproj | 14 ++ .../AgentsWebAPI-CSharp.http | 6 + .../McpServer-CSharp.csproj.in | 33 +++++ .../src/Agents/AgentsWebAPI-CSharp/Program.cs | 41 ++++++ .../src/Agents/AgentsWebAPI-CSharp/README.md | 85 +++++++++++ .../appsettings.Development.json | 8 ++ .../AgentsWebAPI-CSharp/appsettings.json | 9 ++ .../ChatWithCustomData-CSharp.AppHost.csproj | 32 +++++ ...thCustomData-CSharp.ServiceDefaults.csproj | 22 +++ .../ChatWithCustomData-CSharp.Web.csproj | 60 ++++++++ .../Directory.Build.targets | 21 +++ .../ChatWithCustomData/Directory.Build.props | 5 + .../Directory.Build.targets | 15 ++ .../src/ChatWithCustomData/NuGet.config | 25 ++++ .../McpServer-CSharp/McpServer-CSharp.csproj | 33 +++++ 22 files changed, 560 insertions(+), 4 deletions(-) create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/dotnetcli.host.json create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide.host.json create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide/icon.ico create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/template.json create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/AgentsWebAPI-CSharp.csproj create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/AgentsWebAPI-CSharp.http create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/McpServer-CSharp.csproj.in create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/Program.cs create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/README.md create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.Development.json create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.json create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.ServiceDefaults/ChatWithCustomData-CSharp.ServiceDefaults.csproj create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Directory.Build.targets create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.props create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config create mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/McpServer/McpServer-CSharp/McpServer-CSharp.csproj diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs index 38689a97ed7..04a54a82147 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs @@ -22,5 +22,5 @@ public abstract class RetrievalQueryProcessor /// The retrieval query to process. /// The token to monitor for cancellation requests. /// The processed query. - public abstract Task ProcessQueryAsync(RetrievalQuery query, CancellationToken cancellationToken = default); + public abstract Task ProcessAsync(RetrievalQuery query, CancellationToken cancellationToken = default); } diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs index f63639028a0..264eef1a960 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs @@ -23,5 +23,5 @@ public abstract class RetrievalResultProcessor /// The original query (for context during processing). /// The token to monitor for cancellation requests. /// The processed results. - public abstract Task ProcessResultsAsync(RetrievalResults results, RetrievalQuery query, CancellationToken cancellationToken = default); + public abstract Task ProcessAsync(RetrievalResults results, RetrievalQuery query, CancellationToken cancellationToken = default); } diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs index 88bf29e7a9b..6d8e4e0820f 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs @@ -97,7 +97,7 @@ public async Task RetrieveAsync( foreach (var processor in QueryProcessors) { _logger?.RunningQueryProcessor(processor.GetType().Name); - retrievalQuery = await processor.ProcessQueryAsync(retrievalQuery, cancellationToken).ConfigureAwait(false); + retrievalQuery = await processor.ProcessAsync(retrievalQuery, cancellationToken).ConfigureAwait(false); } rootActivity?.SetTag("rag.query.variants", retrievalQuery.Variants.Count); @@ -144,7 +144,7 @@ public async Task RetrieveAsync( foreach (var processor in ResultProcessors) { _logger?.RunningResultProcessor(processor.GetType().Name); - results = await processor.ProcessResultsAsync(results, retrievalQuery, cancellationToken).ConfigureAwait(false); + results = await processor.ProcessAsync(results, retrievalQuery, cancellationToken).ConfigureAwait(false); } rootActivity?.SetTag("rag.results.final_count", results.Chunks.Count); diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/dotnetcli.host.json b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/dotnetcli.host.json new file mode 100644 index 00000000000..5be51dd6357 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/dotnetcli.host.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json.schemastore.org/dotnetcli.host", + "symbolInfo": {}, + "usageExamples": [ + "" + ] +} \ No newline at end of file diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide.host.json b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide.host.json new file mode 100644 index 00000000000..5edf447bbd4 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide.host.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://json.schemastore.org/ide.host", + "order": 0, + "icon": "ide/icon.ico", + "symbolInfo": [] +} diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide/icon.ico b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/.template.config/ide/icon.ico new file mode 100644 index 0000000000000000000000000000000000000000..954709ffd6b9d360fbc0b121a63a29657e0fe768 GIT binary patch literal 38045 zcmeHw2Rzo@`~T;@Z3&f9vPy&OkzGher6|b`?M^i zV@yX+B6rEbjr-L5RnFBEC9E)Jw87-%kE{2ta_(i7l5u%OcG%L0w-XH6PA!zwwPG(T zHsU?2FVlUE&{2O`f8Hbu+J5OR+I|A3&n8G8m3_IsTi>XtZIhBqj>}fLAKNt5(TG?6 zE%!PgkRPy;~cexm9i$*P8P7!=!-Bo!NezYJO20X`I&w@V6yBjDOhNE9RnqUQM2L(1+Ld zJBHHDsrv>>ujkn3`xb^sxCbvgIWA_s(^4_ENs{+%%?F1yd~nk*dsLGh^>D-F`Iii@ z(+-74#*4~oPOOcN9@(@pD$iWs-O)TaE&qw_b1lo=hu^(+TT@JVUG2YOy-sz(Ax?`W zV<%nnEZ|zY=*|gYe(6h=qU!J0zAMvOKhxVwes^@6)8MVm&9gLL%#74%8B?*h8}Il? z`q+CpZ|?*!!WT={PctnYqBcuZ?3_!jL`s9%l$F9-s~^#GVsG`Ed~d(9nn~N;aUarm z51UdJ;^UvAb2Cvyq9mk@b|(3VkK`;Uv1eUniHC%bB9LJD++E(R0sNR zYLOirX}qfN=1?(nT3RBN)4Ht9EmnC=vwOC&cFsAqulDirU~M(dON$1YwOv2wC~~>bx3-sm0d+Op*#7dN6s~&VCwU@kF0X#d z6WT{_sP@DtkAp4rgE_sqg~vaT2wv*>%zSY5EuCta{uaJV7J4sSMOhaJJ)Ykzd2ISr z+sZU$ClB3yUNW1G4y}FPI{&?xoOP0wnqjUsBkX?0GS_g;a&OgsY>o-NZd`bB(e0Ml z%)F+aYd1Yxvtftt0w3Fo4fi<5-*b&xloCGE_4xzk5xESZ7whus4)4hK;w+cnH8+U+ zzVtKAGv=1>c8kQGu!&eUrcrHE^{e&ff_2SP4k;Bnysq&cx-2l~cCRWmBj2Hgx-YWI zw+&gaaJc%e**fwg&sPtcwBLW-wXqr#<$7K85u3SUXrC78i@De3lZtu`5t^~m)6BlA zbXC^G10GUo3ghn;=ISW!^Ek2l%!3D#&*EAX%w)2+HPDI=&TW>sGxS-^tVJtSO*Q28 zaIeAjH(R;NJ{ma8RM_MQ4 zPg_6do&e>`#-r-^c3%YFNq4&~I~HAdBpLCJO`oU0{oU4Twj8fm?Q+>?9%uC9D!BO9 zuB(Vy9&Q(Rc+}zpIe|xfuEy@uPgj%+w#%>+wh^$W>kVj;X>i_b__`)4GCk3>R{EJA zHMwP^e(l{4o-Y>MRwZ+_H2=KhMVhPLEn&E2 z)^CuUUg$I-^3||-Gbx2AJ~rrB)?2a~D;}geQCuzcSzGm`eTJs#OKb zh843W%Wl0_I6(j81T}m1Jw2-mH^7;lW_p+?yGc7=w3Kw& zb?dUS{DP4iyx?Ysc}y% zx^P$4po0ZL9Q>iT*op_;JawsT-Lt)w>WQJxGG7ZVi%3q^c{1YsuoKJIG|syh`u=V| z3B3%Zdq)}PbL7ik3_2LeWg+=NP03B!LM2*##tQcn?$)}`8`m~%dvi2r;C6$(&8rh? zHO4KnvaDys#T{B2uOTj)I4yd5sKtS}Ak%|WT2mLC^zfL~VzXVodycn8lK9?+m_WV= zt-(jF8MF|kYdZv0S{b_XaSgNcbcei-ySP6t!Zd1XYbx9F9G_E1WVV%!%SiX>y?%x8 zIz3r=>5E5g8R8*s!IrQ07L~ta@0XH!=ZMPm#u-twT2sBYYrYzw%Wyh%y~$O3l1;7Q zg7<8d51j%8ZfWc=nWkIf;k0$hkriuml12q4)z=uGI-Q-bO>Z`cdZ^}ARryG}_fdOB zpO8g%1}k?ZGxmioXS|e(*{rO}7oSwm;5lR;;yPPoz#xyo8;l29UvAr2w@R76SDIy! zuv=@y1X|4HM46Bm@^`(*&kc0A>^?Y>bBErEJ5AhiC-s8tLZh!duXy$P_3QHUN%pHI z^|!OkEmswKsS!QZ-Ha*}`yAw#uhg)J@SR*A(7Q^kCOp%o!T+eq#Q9Gb`Q$5mrWIAbZ!6l6 zR8!#?`e9wpq(q-%%}0j6_g$5LvGwfTdY&1NcGPdzJfB`NJ2>_@z&oOM#dq9o-o>$H zBhvc}3~_OKSURmRLbH63#ceP4$GQ8AEqdCS>5S{ijlK44|>VDpQ^~q)vuAdEfVA-W4dg1ioANc zRCZ*W{l%)cW2`fT^aM{BB&}!~u}gfoV|Z|p#kups>T$;zqt4kc5;k2XQ0m1#Lig@t z35)wns0(f7r$r9!wn$ac*s8zGU;q9H(URufSqgi4h&x9FJCA-Ko;Acsj%!_&0LRI? z9v-#w?q-`WOy{}KmaQ-OY|9$6{9OHUB945b{-?|Q+*4W63_#Pf| zv^)vBN99tc4GZ1nwuHEhxzJ{@r0*cJ#hI6eXNQUR@$J>)+BUb=NrE?Q7Wv##_8jB% z#&~h_wz8nHs#zg|y(T`NBb&?Bw?aE}V!%Lpo?_Z;O&8ll&4l9%7xE6apvND$|M-c( z?Znax7Q@b!*PMCae(qHx+ri^)L$reW<;5Ot>-}g(O7go1r|YKmx0i{lFebV`yGsj@ z4oen%RUA2g@6CFX6K$u;7e~uj2W}rLpmmqGW?YP(gktaTne(<++Q&YPo9y6g39-1> ztFGIj!n%EHGO8<7rVHB4In^kg7dx_5Qta8(`ox(VM+ej|zpFiJmgrS=HIqHp>ra?N zCcd_RVI$q6tnUr?Be!Ncs|5DisPtfSRj^8HlzN>0_N1LPY$3LC&(h`RMm_ScvOee& zqMukInQ5~(+bAG3poh8lXq~)RgMlZiAfgL^;_IA=s zq#8utZHf`cg}pu&8(dG74%ob0n082bT%P)YJj%gWy1xIMsH(ebCO;EcEyr<|L0{y5 z|1uS$x4qe*A|;%^!nan@Wuxb=SKfYHY4jK;@8(l+H&)n{PpO@uJEKN!$2-Gw|e&pd6js(Ah;^nsPzcNrx?~hC@J{^ZT>DNp5o>m*#dGI zL$dVq8))t)_{ZH@c(eY*WIYSN^+8KjT-oEJMy912#OS?$S3~1jcA;cON&JCCBNwke zH|k}hC2ZCW=PTWsP-k{VpsabiIYg0C+F(72E1Qm+%h@Nk+2s26J7n(=l2$v@Vyd!5 zR((d8PZ7t0_`q3>>e(f$9xoQilbGKuJ8^bO>}2OnyY1r(&jwZn%eN}hmz!POaLkG~ zO=%CsedtMFrHeD%GF~r^E%G;CI-6_~mTxxXk*j z9ewU!_6S@tV0z0C1IcF>*G!-VJv{u9OZ?i2fQCL7e2z^!RQ$kyp;7UD0jj1n?(~fn z@wt9MD_2SI+WBhv-Amidrz4%cE9aDhuV2qA;fAus@lO>W>oN-F?rupCx7AX%X-##| zb*)NwYn3q1FT1-t|LyK70o4;{3>h2uBrkohlOOYN)Uobjx`xLOZ;72fd}f~yZyFsZ z&fnm8>wU`JEcKT%c@jOI)`o{gUn;Rl@bJEO>3wyWUFp3cqw|BhU3K2s60@{+yrT0H zgBT~Pr<*rchl?jVnnh*C*sU(BDL#j2#R^GtVe7udON z?lk4o6V*lXVztAj1$su=FVa}`;k(*@rVXY(<}prza_LzQX(~L~k&%6(W2i9a@vouowEJyLK52`q zHxnK!OF6GAEM=?nbC=4Cov4rNkUhW}>0|5e3=5`N5zkYymuv z*q%>2JAADS4UZqv+hAu|ra*so>7*muNxgomCms9s^s5S`iZ-!<-AoHtpY* z*d|@hbMpNAojo1}i3`{8pKjBTblEnp*<3PDBJI^xmnzM0cGIY>Y5InK+@)^&uF1Vo z+gcSo?~c-lZZ~4z+K%KAbQ#`!HzZG@{1JY$=8$D(fuxaTR45~`H`7k!8I=gZ&++wQ8y(lU;RQG_gTIt z?#aqQJqxbtHQos22)sJmz_m=(q0f{)e0n`=UNj#Ju%goaGXnw-OC{GByf`1#Or!U^ zsUobI{Nd4T?c!#s8@vnRyPulns(odj<2j3#Ub&|yv#WST#E%lS4Hdn9=5D=-5WS+D zOVsaeW%McEIpg;?UT8f!*ERJ5m)o7Vrz1^nojB8&|G=%4w{~QT@N4(8j=XJEGo0tW zth~MSwvdyj+yot$rh7JAlw`=8qh-h9>Vuj}xUCgk#~Ql5jX&me#DimO_MP)h!M$zU zB=d5mICwn69#5!W<+w)Ch|3{r<^Idp_If;i=lynt8mD8Zn%NWe*;C>Z^m20)OMITW z4sDtFU@QMg?e2{a!avLke8xHH;=O}B^oobi8+fPPiE?_+Eib*g=TQ!hRoCaYz6|a; zm8;^g&>=3*!Ge`n>t%1wo+A|LZkOm&-LvKXRtMjJ{gZj!S|=I~P8qH@HKO0d)(Qt- zwKu)z$#+ZRGhY%@p6?%YGBOc{ns7jTX|q)7ypTmcKIeCODx5jIbGgW?y?f1*1_uns z9ro4lbli=t;#~QTwDAWfOu680>Xcz48>4sN;^qeV+_cw`weQ##-@m46>2>$*kR{$1 z+BTi*cci{*iTlmR9QBerqYr76Tb;~&Q>CGrx4z1HQ(B{pjE0U#SK-or;T6k59Ccz20gtVR0Jx)BjC`(ool&JCOz3 zHxzxKj|dleRgkIkBH)-(sIsO!*wcb-Uq$|+-129Ym8IhsISy$)Aol7-F%o+mjs>wJbksAn{ZZa)0%u=f~4(TsJm719btx(Wy7?XGNj zog`N{)w-dX?t9Z#P0h0*?!dk!R&n>5me5n{=ZnNR<-#%8P?e|VaQSd?^YHF@v2!o@ zjKB7kTT54aG`P3gm8qB}$S|5E(+3qdPv0VZIxWolwl+gEPh$Ej8qeh=X9fDhr06R! zJ^o<-y5*xW@@K?mvu`}Xkr!(vo+e^qonE@HDp)Isc4K{*!gs?pX^{E%PA*lJ%S0(p9TmO8MyqO^%Nbw!s59b}v`I$uZpFz0dZE|Pd!1~c zM~tJKIS-3R%8O==%(&pS;D{e*FK3;RSB}P3sUBZ-`c7Y=Co)lE*6cg1@-UV|jmw;N z`7h9rhNhZx58>dO!$sYH*=0PAaC`0dFw!a>rOmHNPOoE5~t4(w057lXzmweD0NRW@xmYYDtjDAGy^-czev$ zO+}@WV@``qX|340L-lfHUHr&1YiZ4@c@Gb3xjqS5BGhIx_kvA+@abY>*<8C^lQB)x zOW)*)B|PBQ_U9~^!-`Y$n_OyP%NKd9^d=y5Jp$;|7^4ZJV@5fzk;ZR*9DX=R_?=nw zDXLeukH^Og|M$16P+ zd-v`NVEz;szJ@!&AGD|&Hf)$-zkdCGO5gu>qu;`vjg74twLLJUU_!+=YaMwu~Azs+)?6N~ibzZ#Nhl8_NViMn+~97Z+C}!I^dZ*KsF^ zTU%RG4h{~zP_kHfllp%FcY^gt*i zyrZI`NcEpA3l}b=9zJ}?*1v!Mo#6X+*#~49;En$Y8pzAbBaKRb5I;YE8jVK#pQ8bE zKh>>UHwyZH7#|;B`v1}3|7h@E;e+VtXeuo&jW%)OM1c__MhumdlvI(BkWhgiEA8m$ zD0u$-d0Iq7M7wPNugnD}Cnw6y&5aX2t5)B>eFGrdGI@A-Ucis7AqVu`OUU;u`1OI% zf4b||t>cgE|?POv~N(yuQ;G=5x?Ai0?=lYz5^OtJ!EO5UGn^%)GLfZPBXmIS&ORn-x{dCkHBbDc40(4f@}27}tXc{3AF#P9{- z?ERep_2x=J-P9xpt%v?w5cUnEw5L zfd&|FzmE9B)=QW$VFKsq(W9C8!|q+&5r14{{Fi6|`)y@Mdk6CD_PBB5dXRNI$Uq9V zza;G4>m6zIU!eh!??Z(st|`zZ1Z;`hT~8~@3_@5+2IckWyY z2Rmf;K-jesu+7<6ehQPZ@drAbi5qY(Cu2$uGS0C7!?$nW&M|fB)ZdZ@^XAQ?wr<_Z z0lRG___-8E0p_vW?Afz9z}qbJsVP&YP^(t0;)LCMjD?Tg@QjI20l3V zZ_@y>i^ANcU~3w}9%)}+BRl}#a|hvnA0MCpG!2N4m7Sf<2K#hxN81Cuk!50H(gVkL z@h1=bPJ948bYDkDM*;Gfi5rQ>f!{y3PhNm8q=|p>qvdy^fryC67J`?*zdr?eMAn%# zF@M`*Ekp*8STyGKkD||f?suR8;d9vIfh_zXzlnW1lE^DuW7=o1O=_S6Rgh!$m+&VK zcSRb&2Gxc<&jenz;Qw^+zap`{_w3n2fgWW1XLZ!ekc-a^3=9Tg-SIEu-_>Z4nwrYA zWyXvdBMg0~1ew-{zs{zNZBCNKj}8#ZkC(yKpw?cb&Wd~1u$%uM>?#f$#{v$L}^6&Dvr zM{bP;?AKBj?xaS%^@X;!wkk_M{NcsFaq@4{fY>!yPxJ>q$|T-|oMBG*WBeK#8f2X@ z2;<&v_YmBP41jFh0NaO7{JYL7|0WFxKZ6ccuyZqE*CZgGkcly``rQ5`=dk^gh@5~A z)LE#vxAUuLK=4GT?Xv7geiNMt8}J(9Q4o%-f zC;uM)(EIhkeIJS65L}->f8LoW|EaB?NdscfH#Ie}!M@u=#uS~mAFOjyW1hER?pMJk zh=ZMDIeGGA9>|D4!TPf{E2=8O?Kf~AB?p( zVpdMbom3)t!}ce-;Af%w)pk3R2C(T)A(w<5u{R3xo7i*e;Oq8S6KGKbyj39Iz6#I3 zaicS6K=>K{dKA%FBsYTKk1^E&{&m2KIsW+lxBQu3L6g5j1ITGH*k}h}rx#%kCu8oa zAf`fHy?T`iGW?d$?fEk3w83Pptt*zlLIVOnSu2O_&I3P24C|)di4TA9;6ZBEtXU*x z(+#}-xjmmYW5x`AvX7#xmcL4a&+%ZbC4R}C$NXU0^D9@b{QR{pb^bjXfag~NPd;L! zbc|0Eeh3Tw4jh+Hxe*gOWH zO$3Qs0LONnPwaWba5fRy_#?5*u0o%0(SXQ%$nz%1zGCS0JE$L{e#f$JSojlr9)5ii z*5-sU{#^yNABKbEOkr)X7yPt6824rtUO!S3+#v(6n3h_pb5Uf&~kx-Me?w5sOoU4I2V^ z_5ky|p4jjNUoyur9~y}*0Dg~#oHIpihx12c_gw?a??bU{*)oz7N5?u^U&uRs$oi?U z^`|3#>jIr;f_$O@*e}BIU*7}#eM1@C~cqW7TU-27PgOd zBiE-|;8QK}+4-mYtmoT4z0dg69>dRS~T9E{)Q>j{JgMQR4yB8gfq$_))jG_v_=q5**xS}NMt6lkN0(154_gK|Ry zsQon20P0jxG=RDY4Tx$AFqjQAN-#(RY`};#Kv5EaA5TyKlr%sAaMA!l1P%BR#L#{! zK^P6Bq9P42>lWG-X5B`W`jP7l%6B!h_8a+;psCcyx`_HXFtrqE1`SXj2c`wRGYjdJ zRmgWD$f}t$4jSt_5rh7n??eD5is&}%e`7;_HJ#*3{w`(z{{2i)5o1<`{}u&Zy%j#3 z$nUV9llWPrU|)s?{H|R5WDva!zx$6tcIH!E>kn(MB=UUPVJGlIhc-dC{{6+CUx4Px zlP8(Ibfi0h9kg%4deuhQxIFL+egPzZuL}YjbhIRR<`#jQ1v}0+!Or6)IY(W!kk~RQ z@Uh%kHYlMxVBd)G=+dwgxB(-nDEP+PQ88(cb%I0^JNX%F9n8);@ej$3BL#fEhUMo0 zmkP*#Yv=)*r>AEp_V#m{0`HJLtXhEkC3;|v0c>`HgAKuf#4fvXA?wBE<>fTQAEv<{ zZQslAJK;dqIKV4(>>c?k4qcs>kaJ35J^PzD{2ROk-;5$Q8T`+_umvW;7hMOReFxU$ zwnDeMA||hfdWev*aOWhIYZ_uk@@hstHC$Xn6L6=o?_2R zm*6Gj>5zE1U|+;MNPLjM`?Z7M1pM;xO#xGa1KEG@Z}Jk!6URPIcG$lgLG$+g0f=Y3 zL7XE6u!h6OKZ5!(#-RN^?iC{jFy`OHVaJXg%=KpQ^(gQLQyzYzZzlFTnIh+zm&^z3 zZ=gfRN)Q||KJ9B#9dW>3m|xCIM4pqpbIi|37O!KzK1N=*BFX#fnEwI0jlxtrocsTUiP@3~L}Ce; zx6IftiIrkac-q%+_!YcFa!e7^7y>`(3izQIGSCD0|D0s*lD$);K&Gr<`BX%lQU9)V`q)D7aCK8_& z_KpO`;10P@4$!+6ILw9}$wtQLdpP_T}lW2rlWmkZVJ&m@acldd?S0_KQH8a8KeM5BjCW4hd6B? z2k0bDvQ9?yEabf9mvF$kL8tK&(V2Vq?qvhd2C{S}pv^h@2)x=q6_V zHlaD>WFf}nHnDRdhjwAT-wgABk33skT>OvV?)dUQ;U(A|PvP^3BPK!Y-5&5MZli6c zZi1|t4c~_k@uhCC9q6p_?$~!{{bsfOS9BbqG5q+ukd6IF91OZi2sF=O!H>CX|2^H> zS@4r~ew3FGyCD3c2Yz1(8L2_uC2)Xv(A)Um(s{r(m&0mjwvZI~aMXURU~7Wc*c^zW#fU@I{5PQUbhr|ZR5B7Ne+hxm^EldzFUzNclr-{u-;3n|04swp`b+8$Zz@K6|Q(Xae z7VVmwo0;G0fxpHFzuFjcAOLiZgRDtLeHM1vM#S?B5Z~a1pThL#SnqbV`qr&mBrlSN zZ`E)@w(yW|aT5O%-&*bzP|?;4G)HJ~?Nhz>Q;lpRa=qpf&GG26vEs9R{r=0e>_A)5LN-BNmr!q%SGzcMn3ZOF~${kPQrDJD|9x( zd5kMGG(c*~FIA7C-lY2JJ4;YhQIWo+5~-af1^Wu1RubgqLjqQPBvMIveU1 zb~=rQx{XewR-#j+kE181qU}gZ9zjHyuKYp8*3|e`iE=;H|)UySo5(0)AlUIzR=0e-5%s#fPwif-_C}C#%;g=CCj;{}m(#81dx+L;Xv4V?l5i5WnjaA7S;6n!(-#z=qYcRMQbJ&NPd zb6kkA3Sk~F=X+py`3bQ^$q@{qs)|Hg&%O=K6VAYMbE zzc0c7AM~HV@Dg%?85;o&h$Beye>RhTJLY$g^Uj0;vP%U090yN5X7!Kr{pg0wr;VgP ze0PUbe<+;=1NH*YF`xJ$BSZ$f!FOZmVcmsM<6-_2JrJTjJY`EW)Ii~ zBO$+2&LJ00a4kya{O+eMO)mw2r~t zH3rT~@J+grdx?pOr20$cTQI;DBy)2I!5K1T1Zfz29)*g?Z6=?SJ?a0nd<_QZ4Nk7j6=qh|dc-G8_2bM87qFr5-SpW9(0& zZ*9Wo&{bb}yK^p)Zwz0$bcy7j^akB^pi_)61`2?Shm3h)Vc`#T#9)xr3xrbxUbq;3+!BmbZU&=B5Zlzij*V!K>^~l-VEJ3n z-8#@Ku8^%9Bq!%b3ZWt9DDkD_!7~{cOK;>?(SMX@iGM;0-u1?j0=q&GcHxh7&3p#m z2&TX@L|$k>PTv9i9?uowjFAs;7=mZXc#-c;8{v2M7upyNSaU!Z zrf)`4rwG-_rQQdmT?Yw4sT + + + net9.0 + enable + enable + AgentsWebAPI_CSharp + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/AgentsWebAPI-CSharp.http b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/AgentsWebAPI-CSharp.http new file mode 100644 index 00000000000..bcd332375d5 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/AgentsWebAPI-CSharp.http @@ -0,0 +1,6 @@ +@AgentsWebAPI_CSharp_HostAddress = http://localhost:5003 + +GET {{AgentsWebAPI_CSharp_HostAddress}}/weatherforecast/ +Accept: application/json + +### diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/McpServer-CSharp.csproj.in b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/McpServer-CSharp.csproj.in new file mode 100644 index 00000000000..2eca37df228 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/McpServer-CSharp.csproj.in @@ -0,0 +1,33 @@ + + + + net8.0 + Major + Exe + enable + enable + + + true + McpServer + + + README.md + SampleMcpServer + 0.1.0-beta + AI; MCP; server; stdio + An MCP server using the MCP C# SDK. + + + + + + + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/Program.cs b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/Program.cs new file mode 100644 index 00000000000..ee9d65d633f --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/Program.cs @@ -0,0 +1,41 @@ +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. +// Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi +builder.Services.AddOpenApi(); + +var app = builder.Build(); + +// Configure the HTTP request pipeline. +if (app.Environment.IsDevelopment()) +{ + app.MapOpenApi(); +} + +app.UseHttpsRedirection(); + +var summaries = new[] +{ + "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" +}; + +app.MapGet("/weatherforecast", () => +{ + var forecast = Enumerable.Range(1, 5).Select(index => + new WeatherForecast + ( + DateOnly.FromDateTime(DateTime.Now.AddDays(index)), + Random.Shared.Next(-20, 55), + summaries[Random.Shared.Next(summaries.Length)] + )) + .ToArray(); + return forecast; +}) +.WithName("GetWeatherForecast"); + +app.Run(); + +record WeatherForecast(DateOnly Date, int TemperatureC, string? Summary) +{ + public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); +} diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/README.md new file mode 100644 index 00000000000..cb11ac30eb5 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/README.md @@ -0,0 +1,85 @@ +# MCP Server + +This README was created using the C# MCP server project template. It demonstrates how you can easily create an MCP server using C# and publish it as a NuGet package. + +See [aka.ms/nuget/mcp/guide](https://aka.ms/nuget/mcp/guide) for the full guide. + +Please note that this template is currently in an early preview stage. If you have feedback, please take a [brief survey](http://aka.ms/dotnet-mcp-template-survey). + +## Checklist before publishing to NuGet.org + +- Test the MCP server locally using the steps below. +- Update the package metadata in the .csproj file, in particular the ``. +- Update `.mcp/server.json` to declare your MCP server's inputs. + - See [configuring inputs](https://aka.ms/nuget/mcp/guide/configuring-inputs) for more details. +- Pack the project using `dotnet pack`. + +The `bin/Release` directory will contain the package file (.nupkg), which can be [published to NuGet.org](https://learn.microsoft.com/nuget/nuget-org/publish-a-package). + +## Developing locally + +To test this MCP server from source code (locally) without using a built MCP server package, you can configure your IDE to run the project directly using `dotnet run`. + +```json +{ + "servers": { + "McpServer-CSharp": { + "type": "stdio", + "command": "dotnet", + "args": [ + "run", + "--project", + "" + ] + } + } +} +``` + +## Testing the MCP Server + +Once configured, you can ask Copilot Chat for a random number, for example, `Give me 3 random numbers`. It should prompt you to use the `get_random_number` tool on the `McpServer-CSharp` MCP server and show you the results. + +## Publishing to NuGet.org + +1. Run `dotnet pack -c Release` to create the NuGet package +2. Publish to NuGet.org with `dotnet nuget push bin/Release/*.nupkg --api-key --source https://api.nuget.org/v3/index.json` + +## Using the MCP Server from NuGet.org + +Once the MCP server package is published to NuGet.org, you can configure it in your preferred IDE. Both VS Code and Visual Studio use the `dnx` command to download and install the MCP server package from NuGet.org. + +- **VS Code**: Create a `/.vscode/mcp.json` file +- **Visual Studio**: Create a `\.mcp.json` file + +For both VS Code and Visual Studio, the configuration file uses the following server definition: + +```json +{ + "servers": { + "McpServer-CSharp": { + "type": "stdio", + "command": "dnx", + "args": [ + "", + "--version", + "", + "--yes" + ] + } + } +} +``` + +## More information + +.NET MCP servers use the [ModelContextProtocol](https://www.nuget.org/packages/ModelContextProtocol) C# SDK. For more information about MCP: + +- [Official Documentation](https://modelcontextprotocol.io/) +- [Protocol Specification](https://spec.modelcontextprotocol.io/) +- [GitHub Organization](https://github.com/modelcontextprotocol) + +Refer to the VS Code or Visual Studio documentation for more information on configuring and using MCP servers: + +- [Use MCP servers in VS Code (Preview)](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) +- [Use MCP servers in Visual Studio (Preview)](https://learn.microsoft.com/visualstudio/ide/mcp-servers) diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.Development.json b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.Development.json new file mode 100644 index 00000000000..0c208ae9181 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.json b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.json new file mode 100644 index 00000000000..10f68b8c8b4 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/Agents/AgentsWebAPI-CSharp/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj new file mode 100644 index 00000000000..eef4be85a4a --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj @@ -0,0 +1,32 @@ + + + + + + Exe + net9.0 + enable + enable + true + b2f4f5e9-1083-472c-8c3b-f055ac67ba54 + + + + + + + + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.ServiceDefaults/ChatWithCustomData-CSharp.ServiceDefaults.csproj b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.ServiceDefaults/ChatWithCustomData-CSharp.ServiceDefaults.csproj new file mode 100644 index 00000000000..fdec3e59666 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.ServiceDefaults/ChatWithCustomData-CSharp.ServiceDefaults.csproj @@ -0,0 +1,22 @@ + + + + net9.0 + enable + enable + true + + + + + + + + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj new file mode 100644 index 00000000000..0de8abf55d9 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj @@ -0,0 +1,60 @@ + + + + net9.0 + enable + enable + d5681fae-b21b-4114-b781-48180f08c0c4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Directory.Build.targets b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Directory.Build.targets new file mode 100644 index 00000000000..d321029dd95 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Directory.Build.targets @@ -0,0 +1,21 @@ + + + + + + <_LocalChatTemplateVariant>aspire + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.props b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.props new file mode 100644 index 00000000000..0eb47a5ac25 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.props @@ -0,0 +1,5 @@ + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets new file mode 100644 index 00000000000..c1efd69b469 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets @@ -0,0 +1,15 @@ + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config new file mode 100644 index 00000000000..44c152d4490 --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/McpServer/McpServer-CSharp/McpServer-CSharp.csproj b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/McpServer/McpServer-CSharp/McpServer-CSharp.csproj new file mode 100644 index 00000000000..46b9c83818a --- /dev/null +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/McpServer/McpServer-CSharp/McpServer-CSharp.csproj @@ -0,0 +1,33 @@ + + + + net8.0 + Major + Exe + enable + enable + + + true + McpServer + + + README.md + SampleMcpServer + 0.1.0-beta + AI; MCP; server; stdio + An MCP server using the MCP C# SDK. + + + + + + + + + + + + + + From 7da3abec3e4cfb243945f35bc11235cca4b108d5 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 8 Apr 2026 13:57:04 -0400 Subject: [PATCH 03/12] Add tree-aware search paradigm to RetrievalPipeline Phase 2 now checks query.Metadata['search_paradigm'] == 'TreeTraversal': - Wider search (topK * 3) to capture all tree levels - Groups results by Level metadata (set by TreeIndexProcessor) - Prioritizes leaf chunks with branch/root summaries for context - Falls back to existing flat search when metadata not present Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../RetrievalPipeline.cs | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs index 6d8e4e0820f..9a5bb218341 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs @@ -102,13 +102,16 @@ public async Task RetrieveAsync( rootActivity?.SetTag("rag.query.variants", retrievalQuery.Variants.Count); - // Phase 2: Vector search (one search per variant, merge with RRF) + // Phase 2: Vector search (paradigm-aware) var allChunks = new List(); + bool isTreeTraversal = retrievalQuery.Metadata.TryGetValue("search_paradigm", out var paradigm) + && "TreeTraversal".Equals(paradigm?.ToString(), StringComparison.OrdinalIgnoreCase); + int searchTopK = isTreeTraversal ? topK * 3 : topK; foreach (string variant in retrievalQuery.Variants) { _logger?.SearchingVariant(variant.Length > 80 ? variant[..80] : variant); - var searchResults = collection.SearchAsync(variant, top: topK, cancellationToken: cancellationToken); + var searchResults = collection.SearchAsync(variant, top: searchTopK, cancellationToken: cancellationToken); await foreach (var result in searchResults.ConfigureAwait(false)) { @@ -137,6 +140,14 @@ public async Task RetrieveAsync( allChunks = DeduplicateWithRrf(allChunks); } + // Tree traversal: group by level and apply top-down selection + if (isTreeTraversal) + { + int resultsPerLevel = retrievalQuery.Metadata.TryGetValue("results_per_level", out var rpl) && rpl is int r ? r : 3; + allChunks = ApplyTreeTraversal(allChunks, resultsPerLevel, topK); + rootActivity?.SetTag("rag.search_paradigm", "TreeTraversal"); + } + var results = new RetrievalResults { Chunks = allChunks }; rootActivity?.SetTag("rag.results.count", results.Chunks.Count); @@ -180,4 +191,57 @@ private static List DeduplicateWithRrf(List chun return grouped.Select(x => x.Chunk).ToList(); } + + /// + /// Applies top-down tree traversal: groups chunks by level and returns a + /// prioritized mix of leaf chunks with branch/root summaries for context. + /// + /// + /// Expects chunks to have a "Level" or "level" entry in + /// (set by TreeIndexProcessor during ingestion). Chunks without level metadata are treated as leaves. + /// + private static List ApplyTreeTraversal( + List chunks, int resultsPerLevel, int topK) + { + static int GetLevel(RetrievalChunk chunk) + { + if (chunk.Record.TryGetValue("Level", out var lvl) || chunk.Record.TryGetValue("level", out lvl)) + { + return lvl switch + { + int i => i, + long l => (int)l, + string s when int.TryParse(s, out var parsed) => parsed, + _ => 0 + }; + } + + return 0; // Default to leaf + } + + var byLevel = chunks.GroupBy(GetLevel).ToDictionary(g => g.Key, g => g.OrderByDescending(c => c.Score).ToList()); + + var result = new List(); + + // Leaves (level 0) — primary results + if (byLevel.TryGetValue(0, out var leaves)) + { + result.AddRange(leaves.Take(topK)); + } + + // Branch summaries (level 1) — document-level context + if (byLevel.TryGetValue(1, out var branches)) + { + result.AddRange(branches.Take(resultsPerLevel)); + } + + // Root summaries (level 2) — corpus-level context + if (byLevel.TryGetValue(2, out var roots)) + { + result.AddRange(roots.Take(resultsPerLevel)); + } + + // Score: keep original scores but ensure leaves rank first + return result.OrderByDescending(c => c.Score).Take(topK).ToList(); + } } From a49ce18551b86dee5a11f6a8b4f85c8eee07d2bb Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 8 Apr 2026 15:31:46 -0400 Subject: [PATCH 04/12] Remove template dev files with hardcoded local paths These files contained hardcoded c:\Dev\extensions paths for local package resolution. They are template scaffolding not related to retrieval abstractions and should not be on this branch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Directory.Build.targets | 15 ----------- .../src/ChatWithCustomData/NuGet.config | 25 ------------------- 2 files changed, 40 deletions(-) delete mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets delete mode 100644 src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets deleted file mode 100644 index c1efd69b469..00000000000 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/Directory.Build.targets +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config deleted file mode 100644 index 44c152d4490..00000000000 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/NuGet.config +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - From 58c5117e5fbf8dc27b7d15bbe34c3dccedf61378 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 8 Apr 2026 16:44:30 -0400 Subject: [PATCH 05/12] Update abstractions README with retrieval types documentation Add Retrieval Types section documenting RetrievalQuery, RetrievalChunk, RetrievalResults, RetrievalQueryProcessor, RetrievalResultProcessor, and ISearchReranker. Restructure package description to cover both ingestion and retrieval. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../README.md | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md index 0285f27fb3d..ef91a2dd06f 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md @@ -4,9 +4,28 @@ ## The packages -The [Microsoft.Extensions.DataIngestion.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion.Abstractions) package provides the core exchange types, including [`IngestionDocument`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestiondocument), [`IngestionChunker`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunker-1), [`IngestionChunkProcessor`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkprocessor-1), and [`IngestionChunkWriter`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkwriter-1). Any .NET library that provides document processing capabilities can implement these abstractions to enable seamless integration with consuming code. +The [Microsoft.Extensions.DataIngestion.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion.Abstractions) package provides the core exchange types for both ingestion and retrieval. -The [Microsoft.Extensions.DataIngestion](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion) package has an implicit dependency on the `Microsoft.Extensions.DataIngestion.Abstractions` package. This package enables you to easily integrate components such as enrichment processors, vector storage writers, and telemetry into your applications using familiar dependency injection and pipeline patterns. For example, it provides processors for sentiment analysis, keyword extraction, and summarization that can be chained together in ingestion pipelines. +### Ingestion types + +[`IngestionDocument`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestiondocument), [`IngestionChunker`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunker-1), [`IngestionChunkProcessor`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkprocessor-1), and [`IngestionChunkWriter`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkwriter-1). Any .NET library that provides document processing capabilities can implement these abstractions to enable seamless integration with consuming code. + +### Retrieval types + +The retrieval abstractions are the symmetric counterpart to ingestion — they define how applications query, process, and rank results from vector stores: + +| Type | Description | +|------|-------------| +| `RetrievalQuery` | Query text with support for variants (multi-query expansion) and metadata for inter-processor communication. | +| `RetrievalChunk` | A single retrieved chunk with content, relevance score, and record metadata. | +| `RetrievalResults` | Collection of retrieved chunks with pipeline-level metadata (e.g., CRAG scores, reranking info). | +| `RetrievalQueryProcessor` | Abstract base class for pre-search processors (query expansion, HyDE, adaptive routing). | +| `RetrievalResultProcessor` | Abstract base class for post-search processors (re-ranking, CRAG quality gating). | +| `ISearchReranker` | Interface for re-ranking strategies (LLM-based, cross-encoder, ONNX models). | + +### Implementation package + +The [Microsoft.Extensions.DataIngestion](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion) package has an implicit dependency on the `Microsoft.Extensions.DataIngestion.Abstractions` package. This package provides pipeline orchestrators for both ingestion (`IngestionPipeline`) and retrieval (`RetrievalPipeline`), along with dependency injection, telemetry, and processor chaining. The `RetrievalPipeline` supports query variant deduplication via Reciprocal Rank Fusion (RRF) and tree-aware hierarchical search. ## Which package to reference From f4f24f6508a9a1ee908ac52c771992e9f31748f0 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Mon, 13 Apr 2026 13:23:04 -0400 Subject: [PATCH 06/12] Split retrieval into Microsoft.Extensions.DataRetrieval packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move retrieval types from DataIngestion to new DataRetrieval packages: - Microsoft.Extensions.DataRetrieval.Abstractions: RetrievalQuery, RetrievalChunk, RetrievalResults, RetrievalQueryProcessor, RetrievalResultProcessor, IReranker (renamed from ISearchReranker) - Microsoft.Extensions.DataRetrieval: RetrievalPipeline, RetrievalPipelineOptions, retrieval Log methods Namespace changes from Microsoft.Extensions.DataIngestion to Microsoft.Extensions.DataRetrieval. Retrieval types had zero dependencies on ingestion types — clean split. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../README.md | 23 ++--------- .../Microsoft.Extensions.DataIngestion/Log.cs | 9 ----- .../IReranker.cs} | 4 +- ...tensions.DataRetrieval.Abstractions.csproj | 23 +++++++++++ .../README.md | 40 +++++++++++++++++++ .../RetrievalChunk.cs | 2 +- .../RetrievalQuery.cs | 2 +- .../RetrievalQueryProcessor.cs | 2 +- .../RetrievalResultProcessor.cs | 2 +- .../RetrievalResults.cs | 2 +- .../DiagnosticsConstants.cs | 10 +++++ .../Microsoft.Extensions.DataRetrieval/Log.cs | 21 ++++++++++ .../Microsoft.Extensions.DataRetrieval.csproj | 34 ++++++++++++++++ .../RetrievalPipeline.cs | 5 +-- .../RetrievalPipelineOptions.cs | 2 +- 15 files changed, 141 insertions(+), 40 deletions(-) rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs => Microsoft.Extensions.DataRetrieval.Abstractions/IReranker.cs} (93%) create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/Microsoft.Extensions.DataRetrieval.Abstractions.csproj create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/README.md rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions => Microsoft.Extensions.DataRetrieval.Abstractions}/RetrievalChunk.cs (96%) rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions => Microsoft.Extensions.DataRetrieval.Abstractions}/RetrievalQuery.cs (96%) rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions => Microsoft.Extensions.DataRetrieval.Abstractions}/RetrievalQueryProcessor.cs (95%) rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions => Microsoft.Extensions.DataRetrieval.Abstractions}/RetrievalResultProcessor.cs (96%) rename src/Libraries/{Microsoft.Extensions.DataIngestion.Abstractions => Microsoft.Extensions.DataRetrieval.Abstractions}/RetrievalResults.cs (94%) create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/DiagnosticsConstants.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/Log.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/Microsoft.Extensions.DataRetrieval.csproj rename src/Libraries/{Microsoft.Extensions.DataIngestion => Microsoft.Extensions.DataRetrieval}/RetrievalPipeline.cs (98%) rename src/Libraries/{Microsoft.Extensions.DataIngestion => Microsoft.Extensions.DataRetrieval}/RetrievalPipelineOptions.cs (93%) diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md index ef91a2dd06f..d9f87c737fa 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md +++ b/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/README.md @@ -4,28 +4,11 @@ ## The packages -The [Microsoft.Extensions.DataIngestion.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion.Abstractions) package provides the core exchange types for both ingestion and retrieval. +The [Microsoft.Extensions.DataIngestion.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion.Abstractions) package provides the core exchange types, including [`IngestionDocument`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestiondocument), [`IngestionChunker`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunker-1), [`IngestionChunkProcessor`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkprocessor-1), and [`IngestionChunkWriter`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkwriter-1). Any .NET library that provides document processing capabilities can implement these abstractions to enable seamless integration with consuming code. -### Ingestion types +The [Microsoft.Extensions.DataIngestion](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion) package has an implicit dependency on the `Microsoft.Extensions.DataIngestion.Abstractions` package. This package enables you to easily integrate components such as enrichment processors, vector storage writers, and telemetry into your applications using familiar dependency injection and pipeline patterns. -[`IngestionDocument`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestiondocument), [`IngestionChunker`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunker-1), [`IngestionChunkProcessor`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkprocessor-1), and [`IngestionChunkWriter`](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dataingestion.ingestionchunkwriter-1). Any .NET library that provides document processing capabilities can implement these abstractions to enable seamless integration with consuming code. - -### Retrieval types - -The retrieval abstractions are the symmetric counterpart to ingestion — they define how applications query, process, and rank results from vector stores: - -| Type | Description | -|------|-------------| -| `RetrievalQuery` | Query text with support for variants (multi-query expansion) and metadata for inter-processor communication. | -| `RetrievalChunk` | A single retrieved chunk with content, relevance score, and record metadata. | -| `RetrievalResults` | Collection of retrieved chunks with pipeline-level metadata (e.g., CRAG scores, reranking info). | -| `RetrievalQueryProcessor` | Abstract base class for pre-search processors (query expansion, HyDE, adaptive routing). | -| `RetrievalResultProcessor` | Abstract base class for post-search processors (re-ranking, CRAG quality gating). | -| `ISearchReranker` | Interface for re-ranking strategies (LLM-based, cross-encoder, ONNX models). | - -### Implementation package - -The [Microsoft.Extensions.DataIngestion](https://www.nuget.org/packages/Microsoft.Extensions.DataIngestion) package has an implicit dependency on the `Microsoft.Extensions.DataIngestion.Abstractions` package. This package provides pipeline orchestrators for both ingestion (`IngestionPipeline`) and retrieval (`RetrievalPipeline`), along with dependency injection, telemetry, and processor chaining. The `RetrievalPipeline` supports query variant deduplication via Reciprocal Rank Fusion (RRF) and tree-aware hierarchical search. +> **Note:** Retrieval abstractions (`RetrievalPipeline`, `RetrievalQuery`, `RetrievalQueryProcessor`, etc.) live in the separate [`Microsoft.Extensions.DataRetrieval`](../Microsoft.Extensions.DataRetrieval/) package family. ## Which package to reference diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs index 8fcb78f1b77..58732e8ead7 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/Log.cs @@ -37,14 +37,5 @@ internal static partial class Log [LoggerMessage(8, LogLevel.Error, "Unexpected enricher failure.")] internal static partial void UnexpectedEnricherFailure(this ILogger logger, Exception exception); - - [LoggerMessage(9, LogLevel.Debug, "Running query processor: {processor}.")] - internal static partial void RunningQueryProcessor(this ILogger logger, string processor); - - [LoggerMessage(10, LogLevel.Debug, "Searching variant: {variant}.")] - internal static partial void SearchingVariant(this ILogger logger, string variant); - - [LoggerMessage(11, LogLevel.Debug, "Running result processor: {processor}.")] - internal static partial void RunningResultProcessor(this ILogger logger, string processor); } } diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IReranker.cs similarity index 93% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IReranker.cs index 96a12eb34e0..e135c94cde3 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/ISearchReranker.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IReranker.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Defines a re-ranking strategy for retrieval results. @@ -15,7 +15,7 @@ namespace Microsoft.Extensions.DataIngestion; /// Implementations may use LLM-based scoring, cross-encoder models (e.g., ONNX), /// or other ranking strategies. /// -public interface ISearchReranker +public interface IReranker { /// /// Re-ranks the provided chunks based on their relevance to the query. diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/Microsoft.Extensions.DataRetrieval.Abstractions.csproj b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/Microsoft.Extensions.DataRetrieval.Abstractions.csproj new file mode 100644 index 00000000000..d91c0a770a0 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/Microsoft.Extensions.DataRetrieval.Abstractions.csproj @@ -0,0 +1,23 @@ + + + + $(TargetFrameworks);netstandard2.0 + Microsoft.Extensions.DataRetrieval + Abstractions representing Data Retrieval components for RAG. + RAG + RAG;retrieval;search;reranking + true + preview + false + 75 + 75 + + $(NoWarn);S1694;S2368 + + + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/README.md b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/README.md new file mode 100644 index 00000000000..292d86bfdfd --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/README.md @@ -0,0 +1,40 @@ +# Microsoft.Extensions.DataRetrieval.Abstractions + +Abstractions for building composable retrieval pipelines in .NET RAG (Retrieval-Augmented Generation) applications. The retrieval abstractions are the symmetric counterpart to [`Microsoft.Extensions.DataIngestion`](../Microsoft.Extensions.DataIngestion.Abstractions/) — ingestion writes data in, retrieval reads relevant data out. + +## Core Types + +| Type | Description | +|------|-------------| +| `RetrievalQuery` | Query text with support for variants (multi-query expansion) and metadata for inter-processor communication. | +| `RetrievalChunk` | A single retrieved chunk with content, relevance score, and record metadata. | +| `RetrievalResults` | Collection of retrieved chunks with pipeline-level metadata (e.g., CRAG scores, reranking info). | +| `RetrievalQueryProcessor` | Abstract base class for pre-search processors (query expansion, HyDE, adaptive routing). | +| `RetrievalResultProcessor` | Abstract base class for post-search processors (re-ranking, CRAG quality gating). | +| `IReranker` | Interface for re-ranking strategies (LLM-based, cross-encoder, ONNX models). | + +## Which package to reference + +Libraries that provide implementations of the abstractions (e.g., custom query processors, re-rankers) should reference only `Microsoft.Extensions.DataRetrieval.Abstractions`. + +Applications that need the full pipeline orchestrator (`RetrievalPipeline`) should reference `Microsoft.Extensions.DataRetrieval` instead (which itself references the abstractions). + +## Install the package + +From the command-line: + +```console +dotnet add package Microsoft.Extensions.DataRetrieval.Abstractions --prerelease +``` + +Or directly in the C# project file: + +```xml + + + +``` + +## Feedback & Contributing + +We welcome feedback and contributions in [our GitHub repo](https://github.com/dotnet/extensions). diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalChunk.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalChunk.cs index d7aabdbb1aa..e002b0c80f2 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalChunk.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalChunk.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Represents a single chunk returned from retrieval. diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQuery.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQuery.cs index e50f3f0e691..71d084778eb 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQuery.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQuery.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Represents a retrieval query with optional expanded variants and metadata. diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQueryProcessor.cs similarity index 95% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQueryProcessor.cs index 04a54a82147..e5c9f9bbe2e 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalQueryProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalQueryProcessor.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Processes a retrieval query before vector search is performed. diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResultProcessor.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResultProcessor.cs index 264eef1a960..12a54060247 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResultProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResultProcessor.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Processes retrieval results after vector search is performed. diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResults.cs similarity index 94% rename from src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResults.cs index 61f45d1fdad..2011fbd28f7 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion.Abstractions/RetrievalResults.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/RetrievalResults.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Represents the results of a retrieval operation. diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/DiagnosticsConstants.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/DiagnosticsConstants.cs new file mode 100644 index 00000000000..bf208788b7d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/DiagnosticsConstants.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.DataRetrieval; + +internal static class DiagnosticsConstants +{ + internal const string ActivitySourceName = "Experimental.Microsoft.Extensions.DataRetrieval"; + internal const string ErrorTypeTagName = "error.type"; +} diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/Log.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/Log.cs new file mode 100644 index 00000000000..29fd9351487 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/Log.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Logging; + +#pragma warning disable S109 // Magic numbers should not be used + +namespace Microsoft.Extensions.DataRetrieval +{ + internal static partial class Log + { + [LoggerMessage(0, LogLevel.Debug, "Running query processor: {processor}.")] + internal static partial void RunningQueryProcessor(this ILogger logger, string processor); + + [LoggerMessage(1, LogLevel.Debug, "Searching variant: {variant}.")] + internal static partial void SearchingVariant(this ILogger logger, string variant); + + [LoggerMessage(2, LogLevel.Debug, "Running result processor: {processor}.")] + internal static partial void RunningResultProcessor(this ILogger logger, string processor); + } +} diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/Microsoft.Extensions.DataRetrieval.csproj b/src/Libraries/Microsoft.Extensions.DataRetrieval/Microsoft.Extensions.DataRetrieval.csproj new file mode 100644 index 00000000000..7a4c37421a8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/Microsoft.Extensions.DataRetrieval.csproj @@ -0,0 +1,34 @@ + + + + $(TargetFrameworks);netstandard2.0 + Microsoft.Extensions.DataRetrieval + Data Retrieval utilities for RAG. + RAG + RAG;retrieval;search;reranking + true + false + true + preview + false + 75 + 75 + true + + + + + + + + + + + + + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs similarity index 98% rename from src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs index 9a5bb218341..e7aabe86753 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipeline.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs @@ -11,9 +11,9 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.VectorData; using Microsoft.Shared.Diagnostics; -using static Microsoft.Extensions.DataIngestion.DiagnosticsConstants; +using static Microsoft.Extensions.DataRetrieval.DiagnosticsConstants; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; #pragma warning disable IDE0058 // Expression value is never used #pragma warning disable IDE0063 // Use simple 'using' statement @@ -22,7 +22,6 @@ namespace Microsoft.Extensions.DataIngestion; /// Represents a pipeline for retrieving data from a vector store with pre- and post-processing. /// /// -/// Mirrors design. /// Flow: query → → vector search → → results. /// With an empty processor list, this behaves identically to a raw /// call. diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineOptions.cs similarity index 93% rename from src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineOptions.cs index 70f031f2f8d..2f49c2dc2d3 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/RetrievalPipelineOptions.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineOptions.cs @@ -4,7 +4,7 @@ using System.Diagnostics; using Microsoft.Shared.Diagnostics; -namespace Microsoft.Extensions.DataIngestion; +namespace Microsoft.Extensions.DataRetrieval; /// /// Options for configuring the retrieval pipeline. From b95b5cf76d843c40b7c4703baac66242aff8e6aa Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Mon, 13 Apr 2026 13:29:00 -0400 Subject: [PATCH 07/12] Add READMEs for DataRetrieval packages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../README.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/README.md diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/README.md b/src/Libraries/Microsoft.Extensions.DataRetrieval/README.md new file mode 100644 index 00000000000..2e273797139 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/README.md @@ -0,0 +1,37 @@ +# Microsoft.Extensions.DataRetrieval + +Retrieval pipeline orchestration for .NET RAG (Retrieval-Augmented Generation) applications. Built on the abstractions defined in [`Microsoft.Extensions.DataRetrieval.Abstractions`](../Microsoft.Extensions.DataRetrieval.Abstractions/). + +## Key Components + +| Type | Description | +|------|-------------| +| `RetrievalPipeline` | Orchestrates query processing → vector search → result processing with RRF dedup and tree-aware hierarchical search. | +| `RetrievalPipelineOptions` | Configuration for the pipeline (collection name, top-K, processors). | + +## Features + +- **Multi-query deduplication** — Reciprocal Rank Fusion (RRF) merges results from expanded query variants +- **Tree-aware retrieval** — hierarchical traversal from root summaries to leaf chunks +- **Extensible processors** — plug in custom `RetrievalQueryProcessor` and `RetrievalResultProcessor` implementations +- **OpenTelemetry** — built-in distributed tracing and structured logging + +## Install the package + +From the command-line: + +```console +dotnet add package Microsoft.Extensions.DataRetrieval --prerelease +``` + +Or directly in the C# project file: + +```xml + + + +``` + +## Feedback & Contributing + +We welcome feedback and contributions in [our GitHub repo](https://github.com/dotnet/extensions). From d69e9976372f800b4e3a8a5a2179ffab80a40e4f Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Mon, 4 May 2026 20:54:06 -0400 Subject: [PATCH 08/12] Add exception XML docs to RetrieveAsync per FDG audit Added documentation for ArgumentNullException and ArgumentException thrown by RetrieveAsync, per dotnet/runtime adding-api-guidelines.md requirements. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs index e7aabe86753..fcdd4d34888 100644 --- a/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs @@ -71,6 +71,8 @@ public void Dispose() /// Extracts text content from a record for result processing. /// The token to monitor for cancellation requests. /// The retrieval results. + /// is . + /// is or empty. [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2075", Justification = "Record properties are accessed for diagnostic metadata population only.")] public async Task RetrieveAsync( From 634ef85f863978e1417dea14734244b65e9b0585 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 5 May 2026 09:16:06 -0400 Subject: [PATCH 09/12] Add IRetriever interface and BoundRetriever adapter Introduces IRetriever in DataRetrieval.Abstractions as a simple, vector-store-agnostic contract for retrieval pipelines. BoundRetriever in DataRetrieval adapts a RetrievalPipeline + VectorStoreCollection into IRetriever, enabling DI registration and testability. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../IRetriever.cs | 30 +++++++++ .../BoundRetriever.cs | 61 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IRetriever.cs create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IRetriever.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IRetriever.cs new file mode 100644 index 00000000000..6627caa98f4 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval.Abstractions/IRetriever.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.DataRetrieval; + +/// +/// Defines the contract for a retrieval pipeline that processes queries and returns results. +/// +/// +/// Enables DI registration and testability. Consumers depend on +/// rather than a concrete pipeline implementation, allowing mocking in tests and +/// swappable retrieval strategies. +/// +public interface IRetriever +{ + /// + /// Retrieves results for the specified query. + /// + /// The user query. + /// Maximum number of results to retrieve. + /// The token to monitor for cancellation requests. + /// The retrieval results. + Task RetrieveAsync( + string query, + int topK = 5, + CancellationToken cancellationToken = default); +} diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs new file mode 100644 index 00000000000..40505ee2845 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DataRetrieval; + +/// +/// Binds a to a specific vector store collection, +/// producing an that consumers can depend on without +/// knowing about the underlying vector store. +/// +/// The vector store key type. +/// The vector store record type. +/// +/// Register via DI to enable constructor injection of : +/// +/// services.AddSingleton<IRetriever>(sp => +/// new BoundRetriever<string, MyRecord>( +/// sp.GetRequiredService<RetrievalPipeline>(), +/// sp.GetRequiredService<VectorStoreCollection<string, MyRecord>>(), +/// record => record.Content)); +/// +/// +public sealed class BoundRetriever : IRetriever + where TKey : notnull + where TRecord : class +{ + private readonly RetrievalPipeline _pipeline; + private readonly VectorStoreCollection _collection; + private readonly Func? _contentSelector; + + /// + /// Initializes a new instance of the class. + /// + /// The retrieval pipeline to use. + /// The vector store collection to search. + /// Optional function to extract text content from a record. + public BoundRetriever( + RetrievalPipeline pipeline, + VectorStoreCollection collection, + Func? contentSelector = null) + { + _pipeline = Throw.IfNull(pipeline); + _collection = Throw.IfNull(collection); + _contentSelector = contentSelector; + } + + /// + public Task RetrieveAsync( + string query, + int topK = 5, + CancellationToken cancellationToken = default) + { + return _pipeline.RetrieveAsync(_collection, query, topK, _contentSelector, cancellationToken); + } +} From be5bdc8d476a655a65b85b5c16d55655fb229b23 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 5 May 2026 09:27:41 -0400 Subject: [PATCH 10/12] Rename BoundRetriever to VectorStoreRetriever Better communicates what the type does from a consumer perspective: it retrieves from a vector store. Follows noun-noun compound pattern (StreamReader, ChannelWriter) and distinguishes from future IRetriever implementations (WebSearchRetriever, DatabaseRetriever, etc). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../{BoundRetriever.cs => VectorStoreRetriever.cs} | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) rename src/Libraries/Microsoft.Extensions.DataRetrieval/{BoundRetriever.cs => VectorStoreRetriever.cs} (81%) diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs similarity index 81% rename from src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs rename to src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs index 40505ee2845..e2e0228a17a 100644 --- a/src/Libraries/Microsoft.Extensions.DataRetrieval/BoundRetriever.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs @@ -10,9 +10,8 @@ namespace Microsoft.Extensions.DataRetrieval; /// -/// Binds a to a specific vector store collection, -/// producing an that consumers can depend on without -/// knowing about the underlying vector store. +/// An implementation that retrieves results from a +/// using a . /// /// The vector store key type. /// The vector store record type. @@ -20,13 +19,13 @@ namespace Microsoft.Extensions.DataRetrieval; /// Register via DI to enable constructor injection of : /// /// services.AddSingleton<IRetriever>(sp => -/// new BoundRetriever<string, MyRecord>( +/// new VectorStoreRetriever<string, MyRecord>( /// sp.GetRequiredService<RetrievalPipeline>(), /// sp.GetRequiredService<VectorStoreCollection<string, MyRecord>>(), /// record => record.Content)); /// /// -public sealed class BoundRetriever : IRetriever +public sealed class VectorStoreRetriever : IRetriever where TKey : notnull where TRecord : class { @@ -35,12 +34,12 @@ public sealed class BoundRetriever : IRetriever private readonly Func? _contentSelector; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The retrieval pipeline to use. /// The vector store collection to search. /// Optional function to extract text content from a record. - public BoundRetriever( + public VectorStoreRetriever( RetrievalPipeline pipeline, VectorStoreCollection collection, Func? contentSelector = null) From daace23722e197d5db4a20d9b796909b45904c2f Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 5 May 2026 09:52:27 -0400 Subject: [PATCH 11/12] Rename RetrievalPipeline.RetrieveAsync to ProcessAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns with IngestionPipeline.ProcessAsync — both pipelines 'process' inputs through their stages. Reserves 'RetrieveAsync' exclusively for IRetriever, creating clear vocabulary separation: - Pipeline.ProcessAsync: engine that processes queries through stages - IRetriever.RetrieveAsync: endpoint that retrieves results Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs | 6 +++--- .../VectorStoreRetriever.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs index fcdd4d34888..844b98c7b9c 100644 --- a/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipeline.cs @@ -61,7 +61,7 @@ public void Dispose() public IList ResultProcessors { get; } = []; /// - /// Executes the retrieval pipeline: query processing → vector search → result processing. + /// Processes a query through the retrieval pipeline: query processing → vector search → result processing. /// /// The vector store key type. /// The vector store record type. @@ -75,7 +75,7 @@ public void Dispose() /// is or empty. [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2075", Justification = "Record properties are accessed for diagnostic metadata population only.")] - public async Task RetrieveAsync( + public async Task ProcessAsync( VectorStoreCollection collection, string query, int topK = 5, @@ -87,7 +87,7 @@ public async Task RetrieveAsync( Throw.IfNull(collection); Throw.IfNullOrEmpty(query); - using (Activity? rootActivity = _activitySource.StartActivity("RetrievalPipeline.Retrieve")) + using (Activity? rootActivity = _activitySource.StartActivity("RetrievalPipeline.Process")) { rootActivity?.SetTag("rag.query", query) .SetTag("rag.topK", topK); diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs index e2e0228a17a..f662b5e2f27 100644 --- a/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/VectorStoreRetriever.cs @@ -55,6 +55,6 @@ public Task RetrieveAsync( int topK = 5, CancellationToken cancellationToken = default) { - return _pipeline.RetrieveAsync(_collection, query, topK, _contentSelector, cancellationToken); + return _pipeline.ProcessAsync(_collection, query, topK, _contentSelector, cancellationToken); } } From 8ca8adc04ce5f3625c1e83d79c5b3df52f195ba0 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 5 May 2026 10:38:03 -0400 Subject: [PATCH 12/12] Add AsRetriever() extension method on RetrievalPipeline Adds a convenience extension method that wraps a RetrievalPipeline into a VectorStoreRetriever implementing IRetriever. This improves discoverability and enables pipeline.AsRetriever(...) as a natural terminal operation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../RetrievalPipelineExtensions.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineExtensions.cs diff --git a/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineExtensions.cs b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineExtensions.cs new file mode 100644 index 00000000000..64960df8c9b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataRetrieval/RetrievalPipelineExtensions.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; + +namespace Microsoft.Extensions.DataRetrieval; + +/// +/// Extension methods for . +/// +public static class RetrievalPipelineExtensions +{ + /// + /// Creates an that binds this pipeline to a specific vector store collection. + /// + /// The vector store key type. + /// The vector store record type. + /// The retrieval pipeline. + /// The vector store collection to search. + /// Optional function to extract text content from a record. + /// An that processes queries through this pipeline against the specified collection. + /// + /// This is symmetric with IngestionPipeline.ProcessAsync — both pipelines are engines that + /// operate on a data source. AsRetriever creates a ready-to-use endpoint from the engine. + /// + /// var pipeline = new RetrievalPipeline(loggerFactory: loggerFactory); + /// pipeline.QueryProcessors.Add(new MultiQueryExpander(chatClient)); + /// pipeline.ResultProcessors.Add(new LlmReranker(chatClient)); + /// + /// IRetriever retriever = pipeline.AsRetriever(collection, r => r.Content); + /// var results = await retriever.RetrieveAsync("What are the retention policies?"); + /// + /// + public static IRetriever AsRetriever( + this RetrievalPipeline pipeline, + VectorStoreCollection collection, + Func? contentSelector = null) + where TKey : notnull + where TRecord : class + { + return new VectorStoreRetriever(pipeline, collection, contentSelector); + } +}