From 4c0435486144ae0bdb7434158d912281117168c2 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Feb 2026 15:44:08 -0800 Subject: [PATCH 01/38] Realtime Client Proposal --- .../Realtime/DelegatingRealtimeSession.cs | 79 + .../Realtime/IRealtimeClient.cs | 32 + .../Realtime/IRealtimeSession.cs | 59 + .../Realtime/NoiseReductionOptions.cs | 28 + .../Realtime/RealtimeAudioFormat.cs | 36 + ...timeClientConversationItemCreateMessage.cs | 35 + ...timeClientInputAudioBufferAppendMessage.cs | 32 + ...timeClientInputAudioBufferCommitMessage.cs | 23 + .../Realtime/RealtimeClientMessage.cs | 29 + .../RealtimeClientResponseCreateMessage.cs | 86 + .../Realtime/RealtimeContentItem.cs | 54 + .../Realtime/RealtimeServerErrorMessage.cs | 40 + ...imeServerInputAudioTranscriptionMessage.cs | 53 + .../Realtime/RealtimeServerMessage.cs | 34 + .../Realtime/RealtimeServerMessageType.cs | 148 ++ .../RealtimeServerOutputTextAudioMessage.cs | 56 + .../RealtimeServerResponseCreatedMessage.cs | 85 + ...RealtimeServerResponseOutputItemMessage.cs | 42 + .../Realtime/RealtimeSessionKind.cs | 23 + .../Realtime/RealtimeSessionOptions.cs | 138 ++ .../SemanticVoiceActivityDetection.cs | 21 + .../Realtime/ServerVoiceActivityDetection.cs | 36 + .../Realtime/TranscriptionOptions.cs | 38 + .../Realtime/VoiceActivityDetection.cs | 23 + .../Tools/ToolChoiceMode.cs | 28 + .../UsageDetails.cs | 46 + ....Extensions.AI.Evaluation.Reporting.csproj | 2 +- .../Microsoft.Extensions.AI.OpenAI.csproj | 5 + .../OpenAIClientExtensions.cs | 17 + .../OpenAIRealtimeClient.cs | 80 + .../OpenAIRealtimeSession.cs | 1922 +++++++++++++++++ .../FunctionInvokingChatClient.cs | 408 +--- .../Common/FunctionInvocationHelpers.cs | 68 + .../Common/FunctionInvocationLogger.cs | 55 + .../Common/FunctionInvocationProcessor.cs | 259 +++ .../Common/FunctionInvocationStatus.cs | 17 + .../OpenTelemetryConsts.cs | 61 + .../AnonymousDelegatingRealtimeSession.cs | 44 + .../FunctionInvokingRealtimeSession.cs | 484 +++++ ...nvokingRealtimeSessionBuilderExtensions.cs | 43 + .../Realtime/LoggingRealtimeSession.cs | 305 +++ ...LoggingRealtimeSessionBuilderExtensions.cs | 57 + .../Realtime/OpenTelemetryRealtimeSession.cs | 1174 ++++++++++ ...lemetryRealtimeSessionBuilderExtensions.cs | 78 + .../Realtime/RealtimeSessionBuilder.cs | 112 + ...SessionBuilderRealtimeSessionExtensions.cs | 28 + .../Realtime/RealtimeSessionExtensions.cs | 30 + .../Realtime/RealtimeAudioFormatTests.cs | 44 + .../Realtime/RealtimeClientMessageTests.cs | 198 ++ .../Realtime/RealtimeContentItemTests.cs | 63 + .../Realtime/RealtimeServerMessageTests.cs | 266 +++ .../Realtime/RealtimeSessionOptionsTests.cs | 131 ++ .../TestRealtimeSession.cs | 70 + .../OpenAIRealtimeClientTests.cs | 82 + ...OpenAIRealtimeSessionSerializationTests.cs | 974 +++++++++ .../OpenAIRealtimeSessionTests.cs | 101 + .../Microsoft.Extensions.AI.Tests.csproj | 1 + .../DelegatingRealtimeSessionTests.cs | 245 +++ .../FunctionInvokingRealtimeSessionTests.cs | 686 ++++++ .../Realtime/LoggingRealtimeSessionTests.cs | 516 +++++ .../OpenTelemetryRealtimeSessionTests.cs | 1343 ++++++++++++ .../Realtime/RealtimeSessionBuilderTests.cs | 227 ++ .../RealtimeSessionExtensionsTests.cs | 50 + 63 files changed, 11231 insertions(+), 319 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationStatus.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs new file mode 100644 index 00000000000..5ba84d35734 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides an optional base class for an that passes through calls to another instance. +/// +/// +/// This is recommended as a base type when building sessions that can be chained around an underlying . +/// The default implementation simply passes each call to the inner session instance. +/// +[Experimental("MEAI001")] +public class DelegatingRealtimeSession : IRealtimeSession +{ + /// + /// Initializes a new instance of the class. + /// + /// The wrapped session instance. + /// is . + protected DelegatingRealtimeSession(IRealtimeSession innerSession) + { + InnerSession = Throw.IfNull(innerSession); + } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Gets the inner . + protected IRealtimeSession InnerSession { get; } + + /// + public virtual RealtimeSessionOptions? Options => InnerSession.Options; + + /// + public virtual Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + InnerSession.InjectClientMessageAsync(message, cancellationToken); + + /// + public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => + InnerSession.UpdateAsync(options, cancellationToken); + + /// + public virtual IAsyncEnumerable GetStreamingResponseAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken = default) => + InnerSession.GetStreamingResponseAsync(updates, cancellationToken); + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerSession.GetService(serviceType, serviceKey); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + InnerSession.Dispose(); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs new file mode 100644 index 00000000000..72ddcc41b77 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +/// Represents a real-time client. +/// This interface provides methods to create and manage real-time sessions. +[Experimental("MEAI001")] +public interface IRealtimeClient : IDisposable +{ + /// Creates a new real-time session with the specified options. + /// The session options. + /// A token to cancel the operation. + /// The created real-time session. + Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs new file mode 100644 index 00000000000..b813a681d00 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +/// Represents a real-time session. +/// This interface provides methods to manage a real-time session and to interact with the real-time model. +[Experimental("MEAI001")] +public interface IRealtimeSession : IDisposable +{ + /// Updates the session with new options. + /// The new session options. + /// A token to cancel the operation. + /// A task that represents the asynchronous update operation. + Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default); + + /// + /// Gets the current session options. + /// + RealtimeSessionOptions? Options { get; } + + /// + /// Injects a client message into the session. + /// + /// The client message to inject. + /// A token to cancel the operation. + /// A task that represents the asynchronous injection operation. + /// + /// This method allows for the injection of client messages into the session at any time, which can be used to influence the session's behavior or state. + /// + Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); + + /// Sends real-time messages and streams the response. + /// The sequence of real-time messages to send. + /// A token to cancel the operation. + /// The response messages generated by the session. + /// + /// This method cannot be called multiple times concurrently on the same session instance. + /// + IAsyncEnumerable GetStreamingResponseAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs new file mode 100644 index 00000000000..856947d0a4b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring a real-time session. +/// +[Experimental("MEAI001")] +public enum NoiseReductionOptions +{ + /// + /// No noise reduction applied. + /// + None, + + /// + /// for close-talking microphones. + /// + NearField, + + /// + /// For far-field microphones. + /// + FarField +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs new file mode 100644 index 00000000000..034ce64443c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring real-time audio. +/// +[Experimental("MEAI001")] +public class RealtimeAudioFormat +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeAudioFormat(string type, int sampleRate) + { + Type = type; + SampleRate = sampleRate; + } + + /// + /// Gets or sets the type of audio. For example, "audio/pcm". + /// + public string Type { get; set; } + + /// + /// Gets or sets the sample rate of the audio in Hertz. + /// + /// + /// When constructed via , this property is always set. + /// The nullable type allows deserialized instances to omit the sample rate when the server does not provide one. + /// + public int? SampleRate { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs new file mode 100644 index 00000000000..623eb6d3a2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a conversation item. +/// +[Experimental("MEAI001")] +public class RealtimeClientConversationItemCreateMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The conversation item to create. + /// The optional ID of the previous conversation item to insert the new one after. + public RealtimeClientConversationItemCreateMessage(RealtimeContentItem item, string? previousId = null) + { + PreviousId = previousId; + Item = item; + } + + /// + /// Gets or sets the optional previous conversation item ID. + /// If not set, the new item will be appended to the end of the conversation. + /// + public string? PreviousId { get; set; } + + /// + /// Gets or sets the conversation item to create. + /// + public RealtimeContentItem Item { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs new file mode 100644 index 00000000000..1c797bacac8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for appending audio buffer input. +/// +[Experimental("MEAI001")] + +public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The data content containing the audio buffer data to append. + public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) + { + Content = audioContent; + } + + /// + /// Gets or sets the audio content to append to the model audio buffer. + /// + /// + /// The content should include the audio buffer data that needs to be appended to the input audio buffer. + /// + public DataContent Content { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs new file mode 100644 index 00000000000..e2bc048ff9e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for committing audio buffer input. +/// +[Experimental("MEAI001")] + +public class RealtimeClientInputAudioBufferCommitMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeClientInputAudioBufferCommitMessage() + { + } +} + diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs new file mode 100644 index 00000000000..c0413278264 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message the client sends to the model. +/// +[Experimental("MEAI001")] +public class RealtimeClientMessage +{ + /// + /// Gets or sets the optional event ID associated with the message. + /// This can be used for tracking and correlation purposes. + /// + public string? EventId { get; set; } + + /// + /// Gets or sets the raw representation of the message. + /// This can be used to send the raw data to the model. + /// + /// + /// The raw representation is typically used for custom or unsupported message types. + /// For example, the model may accept a JSON serialized message. + /// + public object? RawRepresentation { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs new file mode 100644 index 00000000000..6f2e850d052 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a response item. +/// +[Experimental("MEAI001")] +public class RealtimeClientResponseCreateMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeClientResponseCreateMessage() + { + } + + /// + /// Gets or sets the list of the conversation items to create a response for. + /// + public IList? Items { get; set; } + + /// + /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// + public RealtimeAudioFormat? OutputAudioOptions { get; set; } + + /// + /// Gets or sets the voice of the output audio. + /// + public string? OutputVoice { get; set; } + + /// + /// Gets or sets a value indicating whether the response should be excluded from the conversation history. + /// + public bool ExcludeFromConversation { get; set; } + + /// + /// Gets or sets the instructions allows the client to guide the model on desired responses. + /// If null, the default conversation instructions will be used. + /// + public string? Instructions { get; set; } + + /// + /// Gets or sets the maximum number of output tokens for the response. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets additional metadata for the message. + /// + public AdditionalPropertiesDictionary? Metadata { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the tool choice mode for the response. + /// + /// + /// If AIFunction or HostedMcpServerTool is specified, this value will be ignored. + /// + public ToolChoiceMode? ToolChoiceMode { get; set; } + + /// + /// Gets or sets the AI function to use for the response. + /// + public AIFunction? AIFunction { get; set; } + + /// + /// Gets or sets the hosted MCP server tool configuration for the response. + /// + public HostedMcpServerTool? HostedMcpServerTool { get; set; } + + /// + /// Gets or sets the AI tools available for generating the response. + /// + public IList? Tools { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs new file mode 100644 index 00000000000..7a3713750fe --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time conversation item. +/// +/// +/// This class is used to encapsulate the details of a real-time item that can be inserted into a conversation, +/// or sent as part of a real-time response creation process. +/// +[Experimental("MEAI001")] +public class RealtimeContentItem +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the conversation item. + /// The role of the conversation item. + /// The contents of the conversation item. + public RealtimeContentItem(IList contents, string? id = null, ChatRole? role = null) + { + Id = id; + Role = role; + Contents = contents; + } + + /// + /// Gets or sets the ID of the conversation item. + /// + /// + /// This ID can be null in case passing Function or MCP content where the ID is not required. + /// The Id only needed of having contents representing a user, system, or assistant message with contents like text, audio, image or similar. + /// + public string? Id { get; set; } + + /// + /// Gets or sets the role of the conversation item. + /// + /// + /// The role not used in case of Function or MCP content. + /// The role only needed of having contents representing a user, system, or assistant message with contents like text, audio, image or similar. + /// + public ChatRole? Role { get; set; } + + /// + /// Gets or sets the content of the conversation item. + /// + public IList Contents { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs new file mode 100644 index 00000000000..b75e91793e1 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server error message. +/// +/// +/// Used with the . +/// +[Experimental("MEAI001")] +public class RealtimeServerErrorMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeServerErrorMessage() + { + Type = RealtimeServerMessageType.Error; + } + + /// + /// Gets or sets the error content associated with the error message. + /// + public ErrorContent? Error { get; set; } + + /// + /// Gets or sets an optional event ID caused the error. + /// + public string? ErrorEventId { get; set; } + + /// + /// Gets or sets an optional parameter providing additional context about the error. + /// + public string? Parameter { get; set; } + +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs new file mode 100644 index 00000000000..6e53133a41e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server message for input audio transcription. +/// +/// +/// Used when having InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed response types. +/// +[Experimental("MEAI001")] +public class RealtimeServerInputAudioTranscriptionMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The type of the real-time server response. + /// + /// The parameter should be InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed. + /// + public RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the index of the content part containing the audio. + /// + public int? ContentIndex { get; set; } + + /// + /// Gets or sets the ID of the item containing the audio that is being transcribed. + /// + public string? ItemId { get; set; } + + /// + /// Gets or sets the transcription text of the audio. + /// + public string? Transcription { get; set; } + + /// + /// Gets or sets the usage details for the transcription. + /// + public UsageDetails? Usage { get; set; } + + /// + /// Gets or sets the error content if an error occurred during transcription. + /// + public ErrorContent? Error { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs new file mode 100644 index 00000000000..a6eb9e03d70 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server response message. +/// +[Experimental("MEAI001")] +public class RealtimeServerMessage +{ + /// + /// Gets or sets the type of the real-time response. + /// + public RealtimeServerMessageType Type { get; set; } + + /// + /// Gets or sets the optional event ID associated with the response. + /// This can be used for tracking and correlation purposes. + /// + public string? EventId { get; set; } + + /// + /// Gets or sets the raw representation of the response. + /// This can be used to hold the original data structure received from the model. + /// + /// + /// The raw representation is typically used for custom or unsupported message types. + /// For example, the model may accept a JSON serialized server message. + /// + public object? RawRepresentation { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs new file mode 100644 index 00000000000..bbe7e78f08b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the type of a real-time response. +/// This is used to identify the response type being received from the model. +/// +[Experimental("MEAI001")] +public enum RealtimeServerMessageType +{ + /// + /// Indicates that the response contains only raw content. + /// + /// + /// This response type is to support extensibility for supporting custom content types not natively supported by the SDK. + /// + RawContentOnly, + + /// + /// Indicates the output of audio transcription for user audio written to the user audio buffer. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionCompleted, + + /// + /// Indicates the text value of an input audio transcription content part is updated with incremental transcription results. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionDelta, + + /// + /// Indicates that the audio transcription for user audio written to the user audio buffer has failed. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionFailed, + + /// + /// Indicates the output text update with incremental results response. + /// + /// + /// The type is used with this response type. + /// + OutputTextDelta, + + /// + /// Indicates the output text is complete. + /// + /// + /// The type is used with this response type. + /// + OutputTextDone, + + /// + /// Indicates the model-generated transcription of audio output updated. + /// + /// + /// The type is used with this response type. + /// + OutputAudioTranscriptionDelta, + + /// + /// Indicates the model-generated transcription of audio output is done streaming. + /// + /// + /// The type is used with this response type. + /// + OutputAudioTranscriptionDone, + + /// + /// Indicates the audio output updated. + /// + /// + /// The type is used with this response type. + /// + OutputAudioDelta, + + /// + /// Indicates the audio output is done streaming. + /// + /// + /// The type is used with this response type. + /// + OutputAudioDone, + + /// + /// Indicates the response has been created. + /// + /// + /// The type is used with this response type. + /// + ResponseDone, + + /// + /// Indicates the response has been created. + /// + /// + /// The type is used with this response type. + /// + ResponseCreated, + + /// + /// Indicates an error occurred while processing the request. + /// + /// + /// The type is used with this response type. + /// + Error, + + /// + /// Indicates that an MCP tool call is in progress. + /// + McpCallInProgress, + + /// + /// Indicates that an MCP tool call has completed. + /// + McpCallCompleted, + + /// + /// Indicates that an MCP tool call has failed. + /// + McpCallFailed, + + /// + /// Indicates that listing MCP tools is in progress. + /// + McpListToolsInProgress, + + /// + /// Indicates that listing MCP tools has completed. + /// + McpListToolsCompleted, + + /// + /// Indicates that listing MCP tools has failed. + /// + McpListToolsFailed, +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs new file mode 100644 index 00000000000..b44b0e969cc --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server message for output text and audio. +/// +[Experimental("MEAI001")] +public class RealtimeServerOutputTextAudioMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class for handling output text delta responses. + /// + /// The type of the real-time server response. + /// + /// The should be , , + /// , , + /// , or . + /// + public RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the index of the content part whose text has been updated. + /// + public int? ContentIndex { get; set; } + + /// + /// Gets or sets the text or audio delta, or the final text or audio once the output is complete. + /// + /// + /// if dealing with audio content, this property may contain Base64-encoded audio data. + /// With , usually will have null Text value. + /// + public string? Text { get; set; } + + /// + /// Gets or sets the ID of the item containing the content part whose text has been updated. + /// + public string? ItemId { get; set; } + + /// + /// Gets or sets the index of the output item in the response. + /// + public int? OutputIndex { get; set; } + + /// + /// Gets or sets the ID of the response. + /// + public string? ResponseId { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs new file mode 100644 index 00000000000..d4a94875b8b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a response item. +/// +/// +/// Used with the and messages. +/// +[Experimental("MEAI001")] +public class RealtimeServerResponseCreatedMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The should be or . + /// + public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// + public RealtimeAudioFormat? OutputAudioOptions { get; set; } + + /// + /// Gets or sets the voice of the output audio. + /// + public string? OutputVoice { get; set; } + + /// + /// Gets or sets the conversation ID associated with the response. + /// + public string? ConversationId { get; set; } + + /// + /// Gets or sets the unique response ID. + /// + public string? ResponseId { get; set; } + + /// + /// Gets or sets the maximum number of output tokens for the response. + /// If 0, the service will apply its own limit. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets additional metadata for the message. + /// + public AdditionalPropertiesDictionary? Metadata { get; set; } + + /// + /// Gets or sets the list of the conversation items included in the response. + /// + public IList? Items { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the status of the response. + /// + public string? Status { get; set; } + + /// + /// Gets or sets the error content of the response, if any. + /// + public ErrorContent? Error { get; set; } + + /// + /// Gets or sets the usage details for the response. + /// + public UsageDetails? Usage { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs new file mode 100644 index 00000000000..f6007e6948d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message representing a new output item added or created during response generation. +/// +/// +/// Used with the and messages. +/// +[Experimental("MEAI001")] +public class RealtimeServerResponseOutputItemMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The should be or . + /// + public RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the unique response ID. + /// + public string? ResponseId { get; set; } + + /// + /// Gets or sets the unique output index. + /// + public int? OutputIndex { get; set; } + + /// + /// Gets or sets the conversation item included in the response. + /// + public RealtimeContentItem? Item { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs new file mode 100644 index 00000000000..05be70bffda --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring a real-time session. +/// +[Experimental("MEAI001")] +public enum RealtimeSessionKind +{ + /// + /// Represent a realtime sessions which process audio, text, or other media in real-time. + /// + Realtime, + + /// + /// Represent transcription only session. + /// + Transcription +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs new file mode 100644 index 00000000000..5b2ede4a7f2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -0,0 +1,138 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// Represents options for configuring a real-time session. +[Experimental("MEAI001")] +public class RealtimeSessionOptions +{ + /// + /// Gets or sets the session kind. + /// + /// + /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. + /// + public RealtimeSessionKind SessionKind { get; set; } = RealtimeSessionKind.Realtime; + + /// + /// Gets or sets the model name to use for the session. + /// + public string? Model { get; set; } + + /// + /// Gets or sets the input audio format for the session. + /// + public RealtimeAudioFormat? InputAudioFormat { get; set; } + + /// + /// Gets or sets the noise reduction options for the session. + /// + public NoiseReductionOptions? NoiseReductionOptions { get; set; } + + /// + /// Gets or sets the transcription options for the session. + /// + public TranscriptionOptions? TranscriptionOptions { get; set; } + + /// + /// Gets or sets the voice activity detection options for the session. + /// + public VoiceActivityDetection? VoiceActivityDetection { get; set; } + + /// + /// Gets or sets the output audio format for the session. + /// + public RealtimeAudioFormat? OutputAudioFormat { get; set; } + + /// + /// Gets or sets the output voice speed for the session. + /// + /// + /// The default value is 1.0, which represents normal speed. + /// + public double VoiceSpeed { get; set; } = 1.0; + + /// + /// Gets or sets the output voice for the session. + /// + public string? Voice { get; set; } + + /// + /// Gets or sets the default system instructions for the session. + /// + public string? Instructions { get; set; } + + /// + /// Gets or sets the maximum number of response tokens for the session. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the tool choice mode for the response. + /// + /// + /// If FunctionToolName or McpToolName is specified, this value will be ignored. + /// + public ToolChoiceMode? ToolChoiceMode { get; set; } + + /// + /// Gets or sets the AI function to use for the response. + /// + /// + /// If specified, the ToolChoiceMode will be ignored. + /// + public AIFunction? AIFunction { get; set; } + + /// + /// Gets or sets the name of the MCP tool to use for the response. + /// + /// + /// If specified, the ToolChoiceMode will be ignored. + /// + public HostedMcpServerTool? HostedMcpServerTool { get; set; } + + /// + /// Gets or sets the AI tools available for generating the response. + /// + public IList? Tools { get; set; } + + /// + /// Gets or sets a value indicating whether to enable automatic tracing for the session. + /// if enabled, will create a trace for the session with default values for the workflow name, group id, and metadata. + /// + public bool EnableAutoTracing { get; set; } + + /// + /// Gets or sets the group ID for tracing. + /// + /// + /// This property is only used if is not set to true. + /// + public string? TracingGroupId { get; set; } + + /// + /// Gets or sets the workflow name for tracing. + /// + /// + /// This property is only used if is not set to true. + /// + public string? TracingWorkflowName { get; set; } + + /// + /// Gets or sets arbitrary metadata to attach to this trace to enable filtering. + /// + /// + /// This property is only used if is not set to true. + /// + public object? TracingMetadata { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs new file mode 100644 index 00000000000..74e0d699fe9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring server voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class SemanticVoiceActivityDetection : VoiceActivityDetection +{ + /// + /// Gets or sets the eagerness level for semantic voice activity detection. + /// + /// + /// Examples of the values are "low", "medium", "high", and "auto". + /// + public string Eagerness { get; set; } = "auto"; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs new file mode 100644 index 00000000000..2a0f67970ab --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring server voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class ServerVoiceActivityDetection : VoiceActivityDetection +{ + /// + /// Gets or sets the idle timeout in milliseconds to detect the end of speech. + /// + public int IdleTimeoutInMilliseconds { get; set; } + + /// + /// Gets or sets the prefix padding in milliseconds to include before detected speech. + /// + public int PrefixPaddingInMilliseconds { get; set; } = 300; + + /// + /// Gets or sets the silence duration in milliseconds to consider as a pause. + /// + public int SilenceDurationInMilliseconds { get; set; } = 500; + + /// + /// Gets or sets the threshold for voice activity detection. + /// + /// + /// A value between 0.0 and 1.0, where higher values make the detection more sensitive. + /// + public double Threshold { get; set; } = 0.5; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs new file mode 100644 index 00000000000..b0287112a3a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring transcription in a real-time session. +/// +[Experimental("MEAI001")] +public class TranscriptionOptions +{ + /// + /// Initializes a new instance of the class. + /// + public TranscriptionOptions(string language, string model, string? prompt = null) + { + Language = language; + Model = model; + Prompt = prompt; + } + + /// + /// Gets or sets the language for transcription. The input language should be in ISO-639-1 (e.g. en). + /// + public string Language { get; set; } + + /// + /// Gets or sets the model name to use for transcription. + /// + public string Model { get; set; } + + /// + /// Gets or sets an optional prompt to guide the transcription. + /// + public string? Prompt { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs new file mode 100644 index 00000000000..b7e63616460 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class VoiceActivityDetection +{ + /// + /// Gets or sets a value indicating whether to create a response when voice activity is detected. + /// + public bool CreateResponse { get; set; } + + /// + /// Gets or sets a value indicating whether to interrupt the response when voice activity is detected. + /// + public bool InterruptResponse { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs new file mode 100644 index 00000000000..6093d3b2add --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines modes that controls which if any tool is called by the model. +/// +[Experimental("MEAI001")] +public enum ToolChoiceMode +{ + /// + /// The model will not call any tool and instead generates a message. + /// + None, + + /// + /// The model can pick between generating a message or calling one or more tools. + /// + Auto, + + /// + /// The model must call one or more tools. + /// + Required +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs index b3edbad5e99..a82d1a4f9f1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -38,6 +40,38 @@ public class UsageDetails /// public long? ReasoningTokenCount { get; set; } + /// Gets or sets the number of audio input tokens used. + /// + /// This property is used only when audio input tokens are involved. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? InputAudioTokenCount { get; set; } + + /// Gets or sets the number of text input tokens used. + /// + /// This property is used only when having audio and text tokens. Otherwise InputTokenCount is sufficient. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? InputTextTokenCount { get; set; } + + /// Gets or sets the number of audio output tokens used. + /// + /// This property is used only when audio output tokens are involved. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? OutputAudioTokenCount { get; set; } + + /// Gets or sets the number of text output tokens used. + /// + /// This property is used only when having audio and text tokens. Otherwise OutputTokenCount is sufficient. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? OutputTextTokenCount { get; set; } + /// Gets or sets a dictionary of additional usage counts. /// /// All values set here are assumed to be summable. For example, when middleware makes multiple calls to an underlying @@ -57,6 +91,8 @@ public void Add(UsageDetails usage) TotalTokenCount = NullableSum(TotalTokenCount, usage.TotalTokenCount); CachedInputTokenCount = NullableSum(CachedInputTokenCount, usage.CachedInputTokenCount); ReasoningTokenCount = NullableSum(ReasoningTokenCount, usage.ReasoningTokenCount); + InputAudioTokenCount = NullableSum(InputAudioTokenCount, usage.InputAudioTokenCount); + InputTextTokenCount = NullableSum(InputTextTokenCount, usage.InputTextTokenCount); if (usage.AdditionalCounts is { } countsToAdd) { @@ -109,6 +145,16 @@ internal string DebuggerDisplay parts.Add($"{nameof(ReasoningTokenCount)} = {reasoning}"); } + if (InputAudioTokenCount is { } inputAudio) + { + parts.Add($"{nameof(InputAudioTokenCount)} = {inputAudio}"); + } + + if (InputTextTokenCount is { } inputText) + { + parts.Add($"{nameof(InputTextTokenCount)} = {inputText}"); + } + if (AdditionalCounts is { } additionalCounts) { foreach (var entry in additionalCounts) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj index 8ee31bc2b1a..8a960fc4df1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj @@ -1,6 +1,6 @@  - + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + \ No newline at end of file From a6b4ccdd168c513acd7497323af1a2b951bfb349 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 13 Feb 2026 16:26:20 -0800 Subject: [PATCH 17/38] Replace string Eagerness with SemanticEagerness struct (ChatRole pattern) --- .../Realtime/SemanticEagerness.cs | 95 +++++++++++++++++++ .../SemanticVoiceActivityDetection.cs | 5 +- .../OpenAIRealtimeSession.cs | 4 +- ...OpenAIRealtimeSessionSerializationTests.cs | 2 +- 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs new file mode 100644 index 00000000000..3d6cabe4950 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the eagerness level for semantic voice activity detection. +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct SemanticEagerness : IEquatable +{ + /// Gets a value representing low eagerness. + public static SemanticEagerness Low { get; } = new("low"); + + /// Gets a value representing medium eagerness. + public static SemanticEagerness Medium { get; } = new("medium"); + + /// Gets a value representing high eagerness. + public static SemanticEagerness High { get; } = new("high"); + + /// Gets a value representing automatic eagerness detection. + public static SemanticEagerness Auto { get; } = new("auto"); + + /// + /// Gets the value associated with this . + /// + public string Value { get; } + + /// + /// Initializes a new instance of the struct with the provided value. + /// + /// The value to associate with this . + [JsonConstructor] + public SemanticEagerness(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// + /// Returns a value indicating whether two instances are equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + public static bool operator ==(SemanticEagerness left, SemanticEagerness right) + { + return left.Equals(right); + } + + /// + /// Returns a value indicating whether two instances are not equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + public static bool operator !=(SemanticEagerness left, SemanticEagerness right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is SemanticEagerness other && Equals(other); + + /// + public bool Equals(SemanticEagerness other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override SemanticEagerness Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, SemanticEagerness value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs index 3a406cdda27..12f995d5b42 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -15,8 +15,5 @@ public class SemanticVoiceActivityDetection : VoiceActivityDetection /// /// Gets or sets the eagerness level for semantic voice activity detection. /// - /// - /// Examples of the values are "low", "medium", "high", and "auto". - /// - public string Eagerness { get; set; } = "auto"; + public SemanticEagerness Eagerness { get; set; } = SemanticEagerness.Auto; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 09be4c82fea..e2716c2262c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -190,7 +190,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken ["type"] = "semantic_vad", ["create_response"] = semanticVad.CreateResponse, ["interrupt_response"] = semanticVad.InterruptResponse, - ["eagerness"] = semanticVad.Eagerness, + ["eagerness"] = semanticVad.Eagerness.Value, }; } @@ -1505,7 +1505,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && eagernessElement.GetString() is string eagerness) { - semanticVad.Eagerness = eagerness; + semanticVad.Eagerness = new SemanticEagerness(eagerness); } options.VoiceActivityDetection = semanticVad; diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs index 636652900a5..da5eb80bfc7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs @@ -270,7 +270,7 @@ public async Task ProcessServerEvent_SessionUpdated_WithSemanticVad() var semanticVad = Assert.IsType(options.VoiceActivityDetection); Assert.False(semanticVad.CreateResponse); Assert.True(semanticVad.InterruptResponse); - Assert.Equal("high", semanticVad.Eagerness); + Assert.Equal(SemanticEagerness.High, semanticVad.Eagerness); } [Fact] From 03d76488bdd18d9fa1e7686456f52b2bcee64c12 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 09:27:25 -0800 Subject: [PATCH 18/38] Remove InternalsVisibleToTest from Microsoft.Extensions.AI.OpenAI - Make OpenAIRealtimeSession constructor and ConnectAsync public - Remove InternalsVisibleToTest from csproj - Remove OpenAIRealtimeSessionSerializationTests (depended on internal ConnectWithWebSocketAsync) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Extensions.AI.OpenAI.csproj | 4 - .../OpenAIRealtimeSession.cs | 4 +- ...OpenAIRealtimeSessionSerializationTests.cs | 974 ------------------ 3 files changed, 2 insertions(+), 980 deletions(-) delete mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj index 656f149df95..89ff9b90c29 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj @@ -31,10 +31,6 @@ true - - - - diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index e2716c2262c..cc078142c6a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -62,7 +62,7 @@ public sealed class OpenAIRealtimeSession : IRealtimeSession /// Initializes a new instance of the class. /// The API key used for authentication. /// The model to use for the session. - internal OpenAIRealtimeSession(string apiKey, string model) + public OpenAIRealtimeSession(string apiKey, string model) { _apiKey = apiKey; _model = model; @@ -72,7 +72,7 @@ internal OpenAIRealtimeSession(string apiKey, string model) /// Connects the WebSocket to the OpenAI Realtime API. /// The to monitor for cancellation requests. /// if the connection succeeded; otherwise, . - internal async Task ConnectAsync(CancellationToken cancellationToken = default) + public async Task ConnectAsync(CancellationToken cancellationToken = default) { try { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs deleted file mode 100644 index da5eb80bfc7..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs +++ /dev/null @@ -1,974 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Net.WebSockets; -using System.Runtime.CompilerServices; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; -using Xunit; - -#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. -#pragma warning disable SA1204 // Static elements should appear before instance elements -#pragma warning disable S103 // Lines should not be too long - -namespace Microsoft.Extensions.AI; - -/// -/// Tests for the JSON serialization and deserialization logic used by . -/// Uses a channel-backed WebSocket pair to exercise message processing without a real network connection. -/// -public class OpenAIRealtimeSessionSerializationTests : IAsyncLifetime -{ - private readonly ChannelWebSocket _serverWebSocket; - private readonly ChannelWebSocket _clientWebSocket; - private readonly OpenAIRealtimeSession _session; - private readonly CancellationTokenSource _cts = new(); - - public OpenAIRealtimeSessionSerializationTests() - { - var clientToServer = Channel.CreateUnbounded<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)>(); - var serverToClient = Channel.CreateUnbounded<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)>(); - - _clientWebSocket = new ChannelWebSocket(serverToClient.Reader, clientToServer.Writer); - _serverWebSocket = new ChannelWebSocket(clientToServer.Reader, serverToClient.Writer); - _session = new OpenAIRealtimeSession("test-key", "test-model"); - } - - public async Task InitializeAsync() - { - await _session.ConnectWithWebSocketAsync(_clientWebSocket, _cts.Token); - } - - public Task DisposeAsync() - { - _cts.Cancel(); - _session.Dispose(); - _clientWebSocket.Dispose(); - _serverWebSocket.Dispose(); - _cts.Dispose(); - return Task.CompletedTask; - } - - [Fact] - public async Task ProcessServerEvent_ErrorMessage_ParsedCorrectly() - { - var errorJson = """{"type":"error","event_id":"evt_001","error":{"message":"Something went wrong","code":"invalid_request","param":"model"}}"""; - - await SendServerMessageAsync(errorJson); - var msg = await ReadNextServerMessageAsync(); - - var errorMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.Error, errorMsg.Type); - Assert.Equal("evt_001", errorMsg.EventId); - Assert.Equal("Something went wrong", errorMsg.Error?.Message); - Assert.Equal("invalid_request", errorMsg.Error?.ErrorCode); - Assert.Equal("model", errorMsg.Parameter); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionDelta_ParsedCorrectly() - { - var json = """{"type":"conversation.item.input_audio_transcription.delta","event_id":"evt_002","item_id":"item_001","content_index":0,"delta":"Hello world"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionDelta, transcription.Type); - Assert.Equal("evt_002", transcription.EventId); - Assert.Equal("item_001", transcription.ItemId); - Assert.Equal(0, transcription.ContentIndex); - Assert.Equal("Hello world", transcription.Transcription); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionCompleted_UsesTranscriptField() - { - var json = """{"type":"conversation.item.input_audio_transcription.completed","event_id":"evt_003","item_id":"item_002","content_index":1,"transcript":"The full transcription"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionCompleted, transcription.Type); - Assert.Equal("The full transcription", transcription.Transcription); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionFailed_ParsedCorrectly() - { - var json = """{"type":"conversation.item.input_audio_transcription.failed","event_id":"evt_004","item_id":"item_003","error":{"message":"Transcription failed","code":"transcription_error","param":"audio"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionFailed, transcription.Type); - Assert.NotNull(transcription.Error); - Assert.Equal("Transcription failed", transcription.Error.Message); - Assert.Equal("transcription_error", transcription.Error.ErrorCode); - } - - [Fact] - public async Task ProcessServerEvent_OutputAudioTranscriptDelta_ParsedCorrectly() - { - var json = """{"type":"response.output_audio_transcript.delta","event_id":"evt_005","response_id":"resp_001","item_id":"item_004","output_index":0,"content_index":0,"delta":"Hello"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.OutputAudioTranscriptionDelta, outputMsg.Type); - Assert.Equal("evt_005", outputMsg.EventId); - Assert.Equal("resp_001", outputMsg.ResponseId); - Assert.Equal("item_004", outputMsg.ItemId); - Assert.Equal(0, outputMsg.OutputIndex); - Assert.Equal(0, outputMsg.ContentIndex); - Assert.Equal("Hello", outputMsg.Text); - } - - [Fact] - public async Task ProcessServerEvent_OutputAudioDelta_ParsedCorrectly() - { - var json = """{"type":"response.output_audio.delta","event_id":"evt_006","response_id":"resp_002","item_id":"item_005","output_index":0,"content_index":0,"delta":"base64audiodata"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.OutputAudioDelta, outputMsg.Type); - Assert.Equal("base64audiodata", outputMsg.Text); - } - - [Fact] - public async Task ProcessServerEvent_ResponseCreated_ParsedCorrectly() - { - var json = """{"type":"response.created","event_id":"evt_007","response":{"id":"resp_003","conversation_id":"conv_001","status":"in_progress","max_output_tokens":4096,"output_modalities":["text","audio"],"metadata":{"key1":"value1"},"audio":{"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"alloy"}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var responseMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseCreated, responseMsg.Type); - Assert.Equal("evt_007", responseMsg.EventId); - Assert.Equal("resp_003", responseMsg.ResponseId); - Assert.Equal("conv_001", responseMsg.ConversationId); - Assert.Equal("in_progress", responseMsg.Status); - Assert.Equal(4096, responseMsg.MaxOutputTokens); - Assert.NotNull(responseMsg.OutputModalities); - Assert.Equal(new[] { "text", "audio" }, responseMsg.OutputModalities); - Assert.NotNull(responseMsg.Metadata); - Assert.Equal("value1", responseMsg.Metadata["key1"]); - Assert.NotNull(responseMsg.OutputAudioOptions); - Assert.Equal("audio/pcm", responseMsg.OutputAudioOptions.Type); - Assert.Equal(24000, responseMsg.OutputAudioOptions.SampleRate); - Assert.Equal("alloy", responseMsg.OutputVoice); - } - - [Fact] - public async Task ProcessServerEvent_ResponseDone_WithUsageAndOutput_ParsedCorrectly() - { - var json = """{"type":"response.done","event_id":"evt_008","response":{"id":"resp_004","status":"completed","usage":{"input_tokens":100,"output_tokens":50,"total_tokens":150},"output":[{"type":"message","id":"msg_001","role":"assistant","content":[{"type":"input_text","text":"Hello there!"}]}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var responseMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseDone, responseMsg.Type); - Assert.Equal("resp_004", responseMsg.ResponseId); - Assert.Equal("completed", responseMsg.Status); - Assert.NotNull(responseMsg.Usage); - Assert.Equal(100, responseMsg.Usage.InputTokenCount); - Assert.Equal(50, responseMsg.Usage.OutputTokenCount); - Assert.Equal(150, responseMsg.Usage.TotalTokenCount); - Assert.NotNull(responseMsg.Items); - Assert.Single(responseMsg.Items); - Assert.Equal("msg_001", responseMsg.Items[0].Id); - Assert.Equal(ChatRole.Assistant, responseMsg.Items[0].Role); - var textContent = Assert.IsType(responseMsg.Items[0].Contents[0]); - Assert.Equal("Hello there!", textContent.Text); - } - - [Fact] - public async Task ProcessServerEvent_OutputItemAdded_WithFunctionCall_ParsedCorrectly() - { - var json = """{"type":"response.output_item.added","event_id":"evt_010","response_id":"resp_006","output_index":0,"item":{"type":"function_call","id":"fc_001","name":"get_weather","call_id":"call_001","arguments":"{\"city\":\"Seattle\"}"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputItemMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemAdded, outputItemMsg.Type); - Assert.Equal("resp_006", outputItemMsg.ResponseId); - Assert.Equal(0, outputItemMsg.OutputIndex); - Assert.NotNull(outputItemMsg.Item); - Assert.Equal("fc_001", outputItemMsg.Item.Id); - var functionCall = Assert.IsType(outputItemMsg.Item.Contents[0]); - Assert.Equal("call_001", functionCall.CallId); - Assert.Equal("get_weather", functionCall.Name); - Assert.NotNull(functionCall.Arguments); - Assert.Equal("Seattle", functionCall.Arguments["city"]?.ToString()); - } - - [Fact] - public async Task ProcessServerEvent_SessionCreated_UpdatesOptions() - { - var json = """{"type":"session.created","event_id":"evt_011","session":{"type":"realtime","model":"gpt-realtime","instructions":"Be helpful","max_output_tokens":2048,"output_modalities":["text"],"audio":{"input":{"format":{"type":"audio/pcm","rate":16000},"noise_reduction":{"type":"near_field"},"transcription":{"language":"en","model":"whisper-1"},"turn_detection":{"type":"server_vad","create_response":true,"interrupt_response":true,"idle_timeout_ms":5000,"prefix_padding_ms":300,"silence_duration_ms":500,"threshold":0.5}},"output":{"format":{"type":"audio/pcm","rate":24000},"speed":1.0,"voice":"alloy"}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - Assert.NotNull(msg.RawRepresentation); - - var options = _session.Options; - Assert.NotNull(options); - Assert.Equal(RealtimeSessionKind.Realtime, options.SessionKind); - Assert.Equal("gpt-realtime", options.Model); - Assert.Equal("Be helpful", options.Instructions); - Assert.Equal(2048, options.MaxOutputTokens); - Assert.NotNull(options.InputAudioFormat); - Assert.Equal("audio/pcm", options.InputAudioFormat.Type); - Assert.Equal(16000, options.InputAudioFormat.SampleRate); - Assert.Equal(NoiseReductionOptions.NearField, options.NoiseReductionOptions); - Assert.NotNull(options.TranscriptionOptions); - Assert.Equal("en", options.TranscriptionOptions.SpeechLanguage); - Assert.Equal("whisper-1", options.TranscriptionOptions.ModelId); - - var serverVad = Assert.IsType(options.VoiceActivityDetection); - Assert.True(serverVad.CreateResponse); - Assert.True(serverVad.InterruptResponse); - Assert.Equal(5000, serverVad.IdleTimeoutInMilliseconds); - Assert.Equal(300, serverVad.PrefixPaddingInMilliseconds); - Assert.Equal(500, serverVad.SilenceDurationInMilliseconds); - Assert.Equal(0.5, serverVad.Threshold); - - Assert.NotNull(options.OutputAudioFormat); - Assert.Equal("audio/pcm", options.OutputAudioFormat.Type); - Assert.Equal(24000, options.OutputAudioFormat.SampleRate); - Assert.Equal("alloy", options.Voice); - } - - [Fact] - public async Task ProcessServerEvent_SessionUpdated_WithSemanticVad() - { - var json = """{"type":"session.updated","event_id":"evt_012","session":{"type":"transcription","audio":{"input":{"turn_detection":{"type":"semantic_vad","create_response":false,"interrupt_response":true,"eagerness":"high"}}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - var options = _session.Options; - Assert.NotNull(options); - Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); - var semanticVad = Assert.IsType(options.VoiceActivityDetection); - Assert.False(semanticVad.CreateResponse); - Assert.True(semanticVad.InterruptResponse); - Assert.Equal(SemanticEagerness.High, semanticVad.Eagerness); - } - - [Fact] - public async Task ProcessServerEvent_UnknownEventType_ParsedAsRawContentOnly() - { - var json = """{"type":"some.unknown.event","event_id":"evt_013","data":{"key":"value"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - Assert.NotNull(msg.RawRepresentation); - } - - [Fact] - public async Task InjectClientMessage_ResponseCreate_SerializedCorrectly() - { - var message = new RealtimeClientResponseCreateMessage - { - EventId = "client_evt_001", - Instructions = "Be concise", - MaxOutputTokens = 1024, - OutputModalities = new List { "text", "audio" }, - ExcludeFromConversation = true, - OutputVoice = "alloy", - OutputAudioOptions = new RealtimeAudioFormat("audio/pcm", 24000), - Metadata = new AdditionalPropertiesDictionary { ["key1"] = "value1" }, - ToolMode = ChatToolMode.Auto, - }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("response.create", sent["type"]?.GetValue()); - Assert.Equal("client_evt_001", sent["event_id"]?.GetValue()); - var response = sent["response"]!.AsObject(); - Assert.Equal("Be concise", response["instructions"]?.GetValue()); - Assert.Equal(1024, response["max_output_tokens"]?.GetValue()); - Assert.Equal("none", response["conversation"]?.GetValue()); - Assert.Equal("auto", response["tool_choice"]?.GetValue()); - var modalities = response["output_modalities"]!.AsArray(); - Assert.Equal(2, modalities.Count); - Assert.Equal("text", modalities[0]?.GetValue()); - Assert.Equal("audio", modalities[1]?.GetValue()); - Assert.Equal("value1", response["metadata"]!.AsObject()["key1"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_Message_SerializedCorrectly() - { - var contents = new List { new TextContent("Hello") }; - var item = new RealtimeContentItem(contents, "item_001", ChatRole.User); - var message = new RealtimeClientConversationItemCreateMessage(item, "prev_item_001"); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - Assert.Equal("prev_item_001", sent["previous_item_id"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("message", itemObj["type"]?.GetValue()); - Assert.Equal("item_001", itemObj["id"]?.GetValue()); - Assert.Equal("user", itemObj["role"]?.GetValue()); - var contentArray = itemObj["content"]!.AsArray(); - Assert.Single(contentArray); - Assert.Equal("input_text", contentArray[0]!["type"]?.GetValue()); - Assert.Equal("Hello", contentArray[0]!["text"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_FunctionResult_SerializedCorrectly() - { - var functionResult = new FunctionResultContent("call_001", "Sunny, 72F"); - var item = new RealtimeContentItem(new List { functionResult }, "item_002"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("function_call_output", itemObj["type"]?.GetValue()); - Assert.Equal("call_001", itemObj["call_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_FunctionCall_SerializedCorrectly() - { - var functionCall = new FunctionCallContent("call_002", "get_weather", new Dictionary { ["city"] = "Seattle" }); - var item = new RealtimeContentItem(new List { functionCall }, "item_003"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("function_call", itemObj["type"]?.GetValue()); - Assert.Equal("call_002", itemObj["call_id"]?.GetValue()); - Assert.Equal("get_weather", itemObj["name"]?.GetValue()); - Assert.NotNull(itemObj["arguments"]); - } - - [Fact] - public async Task InjectClientMessage_AudioBufferCommit_SerializedCorrectly() - { - await _session.InjectClientMessageAsync(new RealtimeClientInputAudioBufferCommitMessage()); - var sent = await ReadSentMessageAsync(); - Assert.Equal("input_audio_buffer.commit", sent["type"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_AudioBufferAppend_SerializedCorrectly() - { - var audioBytes = new byte[] { 0x01, 0x02, 0x03, 0x04 }; - var dataContent = new DataContent(audioBytes, "audio/pcm"); - var message = new RealtimeClientInputAudioBufferAppendMessage(dataContent); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("input_audio_buffer.append", sent["type"]?.GetValue()); - var audioBase64 = sent["audio"]!.GetValue(); - Assert.NotNull(audioBase64); - var decoded = Convert.FromBase64String(audioBase64); - Assert.Equal(audioBytes, decoded); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_SerializedCorrectly() - { - var rawJson = """{"type":"custom.event","data":"test"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event", sent["type"]?.GetValue()); - Assert.Equal("test", sent["data"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_WithEventId_PreservesEventId() - { - var rawJson = """{"type":"custom.event","data":"test"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson, EventId = "evt_custom_001" }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event", sent["type"]?.GetValue()); - Assert.Equal("evt_custom_001", sent["event_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_WithEventIdInJson_DoesNotOverwrite() - { - var rawJson = """{"type":"custom.event","event_id":"evt_from_json"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson, EventId = "evt_from_property" }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - // The event_id already present in the raw JSON should take precedence. - Assert.Equal("evt_from_json", sent["event_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_JsonObject_SerializedCorrectly() - { - var rawObj = new JsonObject { ["type"] = "custom.event2", ["payload"] = "data" }; - var message = new RealtimeClientMessage { RawRepresentation = rawObj }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event2", sent["type"]?.GetValue()); - Assert.Equal("data", sent["payload"]?.GetValue()); - } - - #region MCP Tool Serialization Tests - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithUrl_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("my-server", new Uri("https://mcp.example.com/api")) - { - ServerDescription = "A test MCP server", - AllowedTools = new List { "search", "lookup" }, - ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire, - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var sessionObj = sent["session"]!.AsObject(); - var toolsArray = sessionObj["tools"]!.AsArray(); - Assert.Single(toolsArray); - - var toolObj = toolsArray[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("my-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("https://mcp.example.com/api", toolObj["server_url"]?.GetValue()); - Assert.Equal("A test MCP server", toolObj["server_description"]?.GetValue()); - Assert.Equal("never", toolObj["require_approval"]?.GetValue()); - Assert.Null(toolObj["connector_id"]); - - var allowedTools = toolObj["allowed_tools"]!.AsArray(); - Assert.Equal(2, allowedTools.Count); - Assert.Equal("search", allowedTools[0]?.GetValue()); - Assert.Equal("lookup", allowedTools[1]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithConnectorId_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("connector-server", "my-connector-id") - { - AuthorizationToken = "test-token-123", - ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var sessionObj = sent["session"]!.AsObject(); - var toolObj = sessionObj["tools"]!.AsArray()[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("connector-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("my-connector-id", toolObj["connector_id"]?.GetValue()); - Assert.Null(toolObj["server_url"]); - Assert.Equal("always", toolObj["require_approval"]?.GetValue()); - - var authObj = toolObj["authorization"]!.AsObject(); - Assert.Equal("test-token-123", authObj["token"]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithHeaders_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("header-server", new Uri("https://mcp.example.com/")); - mcpTool.Headers["X-Custom"] = "custom-value"; - mcpTool.Headers["Authorization"] = "Bearer my-token"; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolObj = sent["session"]!.AsObject()["tools"]!.AsArray()[0]!.AsObject(); - var headersObj = toolObj["headers"]!.AsObject(); - Assert.Equal("custom-value", headersObj["X-Custom"]?.GetValue()); - Assert.Equal("Bearer my-token", headersObj["Authorization"]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_SpecificApproval_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("specific-server", new Uri("https://mcp.example.com/")) - { - ApprovalMode = HostedMcpServerToolApprovalMode.RequireSpecific( - alwaysRequireApprovalToolNames: new List { "delete_file" }, - neverRequireApprovalToolNames: new List { "read_file", "list_files" }), - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolObj = sent["session"]!.AsObject()["tools"]!.AsArray()[0]!.AsObject(); - var approvalObj = toolObj["require_approval"]!.AsObject(); - - var alwaysObj = approvalObj["always"]!.AsObject(); - var alwaysNames = alwaysObj["tool_names"]!.AsArray(); - Assert.Single(alwaysNames); - Assert.Equal("delete_file", alwaysNames[0]?.GetValue()); - - var neverObj = approvalObj["never"]!.AsObject(); - var neverNames = neverObj["tool_names"]!.AsArray(); - Assert.Equal(2, neverNames.Count); - Assert.Equal("read_file", neverNames[0]?.GetValue()); - Assert.Equal("list_files", neverNames[1]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_MixedTools_AIFunctionAndMcpTool_SerializedCorrectly() - { - var aiFunction = AIFunctionFactory.Create(() => "result", "test_func", "A test function"); - var mcpTool = new HostedMcpServerTool("mcp-server", new Uri("https://mcp.example.com/")); - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [aiFunction, mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolsArray = sent["session"]!.AsObject()["tools"]!.AsArray(); - Assert.Equal(2, toolsArray.Count); - Assert.Equal("function", toolsArray[0]!["type"]?.GetValue()); - Assert.Equal("mcp", toolsArray[1]!["type"]?.GetValue()); - } - - [Fact] - public async Task ResponseCreate_HostedMcpServerTool_InToolsList_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("resp-server", new Uri("https://mcp.example.com/")) - { - ServerDescription = "Response-level MCP", - ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire, - }; - - var message = new RealtimeClientResponseCreateMessage - { - Tools = [mcpTool], - }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var responseObj = sent["response"]!.AsObject(); - var toolsArray = responseObj["tools"]!.AsArray(); - Assert.Single(toolsArray); - var toolObj = toolsArray[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("resp-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("https://mcp.example.com/", toolObj["server_url"]?.GetValue()); - Assert.Equal("Response-level MCP", toolObj["server_description"]?.GetValue()); - Assert.Equal("never", toolObj["require_approval"]?.GetValue()); - } - - #endregion - - #region MCP Server Event Parsing Tests - - [Fact] - public async Task ProcessServerEvent_McpCallCompleted_ParsedCorrectly() - { - var json = """{"type":"mcp_call.completed","event_id":"evt_mcp_001","response_id":"resp_010","output_index":0,"item":{"type":"mcp_call","id":"mcp_call_001","name":"search","server_label":"my-mcp","arguments":"{\"query\":\"weather\"}","output":"Sunny, 72F"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallCompleted, outputMsg.Type); - Assert.Equal("evt_mcp_001", outputMsg.EventId); - Assert.Equal("resp_010", outputMsg.ResponseId); - Assert.Equal(0, outputMsg.OutputIndex); - - Assert.NotNull(outputMsg.Item); - Assert.Equal("mcp_call_001", outputMsg.Item.Id); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("mcp_call_001", callContent.CallId); - Assert.Equal("search", callContent.ToolName); - Assert.Equal("my-mcp", callContent.ServerName); - Assert.NotNull(callContent.Arguments); - Assert.Equal("weather", callContent.Arguments["query"]?.ToString()); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - Assert.Equal("mcp_call_001", resultContent.CallId); - Assert.NotNull(resultContent.Output); - var textOutput = Assert.IsType(resultContent.Output[0]); - Assert.Equal("Sunny, 72F", textOutput.Text); - } - - [Fact] - public async Task ProcessServerEvent_McpCallFailed_ParsedWithError() - { - var json = """{"type":"mcp_call.failed","event_id":"evt_mcp_002","item":{"type":"mcp_call","id":"mcp_call_002","name":"delete_file","server_label":"file-server","arguments":"{\"path\":\"/tmp/x\"}","error":{"type":"tool_execution_error","message":"Permission denied"}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallFailed, outputMsg.Type); - - Assert.NotNull(outputMsg.Item); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("delete_file", callContent.ToolName); - Assert.Equal("file-server", callContent.ServerName); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - var errorContent = Assert.IsType(resultContent.Output![0]); - Assert.Contains("Permission denied", errorContent.Message); - } - - [Fact] - public async Task ProcessServerEvent_McpCallInProgress_ParsedCorrectly() - { - var json = """{"type":"mcp_call.in_progress","event_id":"evt_mcp_003","item_id":"mcp_call_003"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallInProgress, outputMsg.Type); - Assert.Equal("evt_mcp_003", outputMsg.EventId); - Assert.NotNull(outputMsg.Item); - Assert.Equal("mcp_call_003", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsCompleted_ParsedWithToolsList() - { - var json = """{"type":"mcp_list_tools.completed","event_id":"evt_mcp_004","item_id":"list_001","item":{"type":"mcp_list_tools","id":"list_001","server_label":"my-mcp","tools":[{"name":"search","description":"Search the web","input_schema":{"type":"object"}},{"name":"lookup","description":"Lookup a value"}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsCompleted, outputMsg.Type); - Assert.Equal("evt_mcp_004", outputMsg.EventId); - Assert.NotNull(outputMsg.RawRepresentation); - - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_001", outputMsg.Item.Id); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var tool1 = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("search", tool1.ToolName); - Assert.Equal("my-mcp", tool1.ServerName); - - var tool2 = Assert.IsType(outputMsg.Item.Contents[1]); - Assert.Equal("lookup", tool2.ToolName); - Assert.Equal("my-mcp", tool2.ServerName); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsInProgress_ParsedWithItemId() - { - var json = """{"type":"mcp_list_tools.in_progress","event_id":"evt_mcp_005","item_id":"list_002"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsInProgress, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_002", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsFailed_ParsedWithItemId() - { - var json = """{"type":"mcp_list_tools.failed","event_id":"evt_mcp_006","item_id":"list_003"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsFailed, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_003", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpApprovalRequest_ParsedCorrectly() - { - var json = """{"type":"conversation.item.added","event_id":"evt_mcp_007","item":{"type":"mcp_approval_request","id":"approval_001","name":"charge_card","server_label":"payment-mcp","arguments":"{\"amount\":99.99}"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.NotNull(outputMsg.Item); - Assert.Single(outputMsg.Item.Contents); - - var approvalRequest = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("approval_001", approvalRequest.Id); - Assert.Equal("charge_card", approvalRequest.ToolCall.ToolName); - Assert.Equal("payment-mcp", approvalRequest.ToolCall.ServerName); - Assert.NotNull(approvalRequest.ToolCall.Arguments); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemDone_WithMcpCall_ParsedCorrectly() - { - var json = """{"type":"conversation.item.done","event_id":"evt_mcp_008","item":{"type":"mcp_call","id":"mcp_done_001","name":"read_file","server_label":"fs-server","arguments":"{\"path\":\"/readme.md\"}","output":"# Hello"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemDone, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("read_file", callContent.ToolName); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - var textOutput = Assert.IsType(resultContent.Output![0]); - Assert.Equal("# Hello", textOutput.Text); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemAdded_WithRegularMessage_ParsedCorrectly() - { - var json = """{"type":"conversation.item.added","event_id":"evt_conv_001","item":{"type":"message","id":"msg_conv_001","role":"user","content":[{"type":"input_text","text":"Hello"}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemAdded, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("msg_conv_001", outputMsg.Item.Id); - Assert.Equal(ChatRole.User, outputMsg.Item.Role); - var textContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("Hello", textContent.Text); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemAdded_WithUnknownType_ReturnsRawContent() - { - var json = """{"type":"conversation.item.added","event_id":"evt_conv_002","item":{"type":"unknown_item_type","id":"unknown_001"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.RawContentOnly, outputMsg.Type); - Assert.NotNull(outputMsg.RawRepresentation); - } - - #endregion - - #region MCP Approval Response Sending Tests - - [Fact] - public async Task InjectClientMessage_McpApprovalResponse_Approved_SerializedCorrectly() - { - var approvalResponse = new McpServerToolApprovalResponseContent("approval_001", approved: true); - var item = new RealtimeContentItem(new List { approvalResponse }, "resp_item_001"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("mcp_approval_response", itemObj["type"]?.GetValue()); - Assert.Equal("approval_001", itemObj["approval_request_id"]?.GetValue()); - Assert.True(itemObj["approve"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_McpApprovalResponse_Rejected_SerializedCorrectly() - { - var approvalResponse = new McpServerToolApprovalResponseContent("approval_002", approved: false); - var item = new RealtimeContentItem(new List { approvalResponse }, "resp_item_002"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("mcp_approval_response", itemObj["type"]?.GetValue()); - Assert.Equal("approval_002", itemObj["approval_request_id"]?.GetValue()); - Assert.False(itemObj["approve"]?.GetValue()); - } - - #endregion - - private async Task SendServerMessageAsync(string json) - { - var bytes = Encoding.UTF8.GetBytes(json); - await _serverWebSocket.SendAsync(new ArraySegment(bytes), WebSocketMessageType.Text, true, CancellationToken.None); - } - - private async Task ReadSentMessageAsync() - { - var buffer = new byte[1024 * 16]; - var result = await _serverWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); - var json = Encoding.UTF8.GetString(buffer, 0, result.Count); - return JsonSerializer.Deserialize(json)!; - } - - private async Task ReadNextServerMessageAsync() - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var enumerator = _session.GetStreamingResponseAsync(EmptyUpdatesAsync(), cts.Token) - .GetAsyncEnumerator(cts.Token); - try - { - if (await enumerator.MoveNextAsync().ConfigureAwait(false)) - { - return enumerator.Current; - } - } - finally - { - await enumerator.DisposeAsync().ConfigureAwait(false); - } - - throw new InvalidOperationException("No server message received within timeout"); - } - - private static async IAsyncEnumerable EmptyUpdatesAsync( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); - yield break; - } - - /// - /// A WebSocket implementation backed by channels, used for in-process testing without a real network connection. - /// - internal sealed class ChannelWebSocket : WebSocket - { - private readonly ChannelReader<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> _reader; - private readonly ChannelWriter<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> _writer; - private WebSocketState _state = WebSocketState.Open; - - public ChannelWebSocket( - ChannelReader<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> reader, - ChannelWriter<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> writer) - { - _reader = reader; - _writer = writer; - } - - public override WebSocketCloseStatus? CloseStatus => _state == WebSocketState.Closed ? WebSocketCloseStatus.NormalClosure : null; - public override string? CloseStatusDescription => null; - public override WebSocketState State => _state; - public override string? SubProtocol => null; - - public override async Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) - { - var (data, type, endOfMessage) = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false); - - if (type == WebSocketMessageType.Close) - { - _state = WebSocketState.CloseReceived; - return new WebSocketReceiveResult(0, WebSocketMessageType.Close, true, WebSocketCloseStatus.NormalClosure, null); - } - - var count = Math.Min(data.Length, buffer.Count); - Array.Copy(data, 0, buffer.Array!, buffer.Offset, count); - return new WebSocketReceiveResult(count, type, endOfMessage); - } - - public override async Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) - { - var data = new byte[buffer.Count]; - Array.Copy(buffer.Array!, buffer.Offset, data, 0, buffer.Count); - await _writer.WriteAsync((data, messageType, endOfMessage), cancellationToken).ConfigureAwait(false); - } - - public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - _state = WebSocketState.Closed; - _writer.TryComplete(); - return Task.CompletedTask; - } - - public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - _state = WebSocketState.CloseSent; - _writer.TryComplete(); - return Task.CompletedTask; - } - - public override void Abort() - { - _state = WebSocketState.Aborted; - _writer.TryComplete(); - } - - public override void Dispose() - { - _state = WebSocketState.Closed; - _writer.TryComplete(); - } - } -} From 5845991cf512b76f167323e85c8f51c878320683 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 09:51:47 -0800 Subject: [PATCH 19/38] Add RawRepresentationFactory to RealtimeSessionOptions Add Func? RawRepresentationFactory property following the same pattern used by ChatOptions, EmbeddingGenerationOptions, and other abstraction options types. Add note in OpenAIRealtimeSession to consume the factory when switching to the OpenAI SDK. --- .../Realtime/RealtimeSessionOptions.cs | 22 +++++++++++++++++++ .../OpenAIRealtimeSession.cs | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index c8c0d3fb236..7e52db231c1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; @@ -136,4 +138,24 @@ public class RealtimeSessionOptions /// This property is only used if is not set to true. /// public object? TracingMetadata { get; set; } + + /// + /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. + /// + /// + /// The underlying implementation might have its own representation of options. + /// When is invoked with a , + /// that implementation might convert the provided options into its own representation in order to use it while + /// performing the operation. For situations where a consumer knows which concrete + /// is being used and how it represents options, a new instance of that implementation-specific options type can be + /// returned by this callback for the implementation to use, instead of creating a + /// new instance. Such implementations might mutate the supplied options instance further based on other settings + /// supplied on this instance or from other inputs. + /// Therefore, it is strongly recommended to not return shared instances and instead make the callback return + /// a new instance on each call. + /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly typed + /// properties on . + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index cc078142c6a..4c0a34b859e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -118,6 +118,10 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { _ = Throw.IfNull(options); + // Note: When switching to the OpenAI SDK for serialization, consume options.RawRepresentationFactory + // here to allow callers to provide a pre-configured SDK-specific options instance, following the + // same pattern used by OpenAIChatClient and other provider implementations. + var sessionElement = new JsonObject { ["type"] = "session.update", From 63531b5e16015e1313fe72de6fd49451ba71fbfb Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 10:09:26 -0800 Subject: [PATCH 20/38] Remove OpenAI-specific tracing properties from RealtimeSessionOptions Remove EnableAutoTracing, TracingGroupId, TracingWorkflowName, and TracingMetadata from the abstraction layer. These are OpenAI-specific and should be configured via RawRepresentationFactory when the OpenAI SDK dependency is added. --- .../Realtime/RealtimeSessionOptions.cs | 30 ------------------- .../OpenAIRealtimeSession.cs | 4 --- .../Realtime/RealtimeSessionOptionsTests.cs | 13 -------- 3 files changed, 47 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 7e52db231c1..0812757c178 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -109,36 +109,6 @@ public class RealtimeSessionOptions /// public IList? Tools { get; set; } - /// - /// Gets or sets a value indicating whether to enable automatic tracing for the session. - /// if enabled, will create a trace for the session with default values for the workflow name, group id, and metadata. - /// - public bool EnableAutoTracing { get; set; } - - /// - /// Gets or sets the group ID for tracing. - /// - /// - /// This property is only used if is not set to true. - /// - public string? TracingGroupId { get; set; } - - /// - /// Gets or sets the workflow name for tracing. - /// - /// - /// This property is only used if is not set to true. - /// - public string? TracingWorkflowName { get; set; } - - /// - /// Gets or sets arbitrary metadata to attach to this trace to enable filtering. - /// - /// - /// This property is only used if is not set to true. - /// - public object? TracingMetadata { get; set; } - /// /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 4c0a34b859e..118f01324e5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1366,10 +1366,6 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati newOptions.AIFunction = Options.AIFunction; newOptions.HostedMcpServerTool = Options.HostedMcpServerTool; newOptions.ToolMode = Options.ToolMode; - newOptions.EnableAutoTracing = Options.EnableAutoTracing; - newOptions.TracingGroupId = Options.TracingGroupId; - newOptions.TracingWorkflowName = Options.TracingWorkflowName; - newOptions.TracingMetadata = Options.TracingMetadata; } Options = newOptions; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index 57c588801b9..f789b4c60dd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -31,10 +31,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.AIFunction); Assert.Null(options.HostedMcpServerTool); Assert.Null(options.Tools); - Assert.False(options.EnableAutoTracing); - Assert.Null(options.TracingGroupId); - Assert.Null(options.TracingWorkflowName); - Assert.Null(options.TracingMetadata); } [Fact] @@ -46,7 +42,6 @@ public void Properties_Roundtrip() var outputFormat = new RealtimeAudioFormat("audio/pcm", 24000); List modalities = ["text", "audio"]; List tools = [AIFunctionFactory.Create(() => 42)]; - var tracingMetadata = new { key = "value" }; var transcriptionOptions = new TranscriptionOptions { SpeechLanguage = "en", ModelId = "whisper-1", Prompt = "greeting" }; var vad = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; @@ -64,10 +59,6 @@ public void Properties_Roundtrip() options.OutputModalities = modalities; options.ToolMode = ChatToolMode.Auto; options.Tools = tools; - options.EnableAutoTracing = true; - options.TracingGroupId = "group-1"; - options.TracingWorkflowName = "workflow-1"; - options.TracingMetadata = tracingMetadata; Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); Assert.Equal("gpt-4-realtime", options.Model); @@ -83,10 +74,6 @@ public void Properties_Roundtrip() Assert.Same(modalities, options.OutputModalities); Assert.Equal(ChatToolMode.Auto, options.ToolMode); Assert.Same(tools, options.Tools); - Assert.True(options.EnableAutoTracing); - Assert.Equal("group-1", options.TracingGroupId); - Assert.Equal("workflow-1", options.TracingWorkflowName); - Assert.Same(tracingMetadata, options.TracingMetadata); } [Fact] From f333eaa7e47193ce68edb32e09ab2c3cf047a4fb Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:03:12 -0800 Subject: [PATCH 21/38] Remove AIFunction/HostedMcpServerTool properties, use ChatToolMode Remove redundant AIFunction and HostedMcpServerTool properties from RealtimeSessionOptions and RealtimeClientResponseCreateMessage. Callers should use ChatToolMode.RequireSpecific(functionName) instead. Update OpenAI serialization to emit structured tool_choice JSON object when RequireSpecific is used. Update OpenTelemetry and tests accordingly. --- .../RealtimeClientResponseCreateMessage.cs | 13 --------- .../Realtime/RealtimeSessionOptions.cs | 19 ------------- .../OpenAIRealtimeSession.cs | 27 +++++-------------- .../Realtime/OpenTelemetryRealtimeSession.cs | 13 +-------- .../Realtime/RealtimeClientMessageTests.cs | 2 -- .../Realtime/RealtimeSessionOptionsTests.cs | 2 -- .../OpenTelemetryRealtimeSessionTests.cs | 14 +++------- 7 files changed, 11 insertions(+), 79 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 07dd8988bce..a840195b4fd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -65,21 +65,8 @@ public RealtimeClientResponseCreateMessage() /// /// Gets or sets the tool choice mode for the response. /// - /// - /// If AIFunction or HostedMcpServerTool is specified, this value will be ignored. - /// public ChatToolMode? ToolMode { get; set; } - /// - /// Gets or sets the AI function to use for the response. - /// - public AIFunction? AIFunction { get; set; } - - /// - /// Gets or sets the hosted MCP server tool configuration for the response. - /// - public HostedMcpServerTool? HostedMcpServerTool { get; set; } - /// /// Gets or sets the AI tools available for generating the response. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 0812757c178..6180a12ab87 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -83,27 +83,8 @@ public class RealtimeSessionOptions /// /// Gets or sets the tool choice mode for the session. /// - /// - /// If or is specified, this value will be ignored. - /// public ChatToolMode? ToolMode { get; set; } - /// - /// Gets or sets the AI function to use for the response. - /// - /// - /// If specified, the will be ignored. - /// - public AIFunction? AIFunction { get; set; } - - /// - /// Gets or sets the name of the MCP tool to use for the response. - /// - /// - /// If specified, the will be ignored. - /// - public HostedMcpServerTool? HostedMcpServerTool { get; set; } - /// /// Gets or sets the AI tools available for generating the response. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 118f01324e5..8266378e3fd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -400,28 +400,15 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel responseObj["output_modalities"] = CreateModalitiesArray(responseCreate.OutputModalities); } - if (responseCreate.AIFunction is not null) - { - responseObj["tool_choice"] = new JsonObject - { - ["type"] = "function", - ["name"] = responseCreate.AIFunction.Name, - }; - } - else if (responseCreate.HostedMcpServerTool is not null) - { - responseObj["tool_choice"] = new JsonObject - { - ["type"] = "mcp", - ["server_label"] = responseCreate.HostedMcpServerTool.ServerName, - ["name"] = responseCreate.HostedMcpServerTool.Name, - }; - } - else if (responseCreate.ToolMode is { } toolMode) + if (responseCreate.ToolMode is { } toolMode) { responseObj["tool_choice"] = toolMode switch { - RequiredChatToolMode r when r.RequiredFunctionName is not null => r.RequiredFunctionName, + RequiredChatToolMode r when r.RequiredFunctionName is not null => new JsonObject + { + ["type"] = "function", + ["name"] = r.RequiredFunctionName, + }, RequiredChatToolMode => "required", NoneChatToolMode => "none", _ => "auto", @@ -1363,8 +1350,6 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati if (Options is not null) { newOptions.Tools = Options.Tools; - newOptions.AIFunction = Options.AIFunction; - newOptions.HostedMcpServerTool = Options.HostedMcpServerTool; newOptions.ToolMode = Options.ToolMode; } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 65122718a2a..a559ce23973 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -808,19 +808,8 @@ private static string SerializeMessages(IEnumerable message } // Tool choice mode (custom attribute - not part of OTel GenAI spec) - // Priority: AIFunction > HostedMcpServerTool > ToolMode string? toolChoice = null; - if (options.AIFunction is { } aiFunc) - { - // When a specific AIFunction is forced, use its name - toolChoice = aiFunc.Name; - } - else if (options.HostedMcpServerTool is { } mcpTool) - { - // When a specific MCP tool is forced, use the server name (mcp:) - toolChoice = $"mcp:{mcpTool.ServerName}"; - } - else if (options.ToolMode is { } toolMode) + if (options.ToolMode is { } toolMode) { toolChoice = toolMode switch { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index 4a5847bd80a..a2d0530d39c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -140,8 +140,6 @@ public void ResponseCreateMessage_DefaultProperties() Assert.Null(message.Metadata); Assert.Null(message.OutputModalities); Assert.Null(message.ToolMode); - Assert.Null(message.AIFunction); - Assert.Null(message.HostedMcpServerTool); Assert.Null(message.Tools); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index f789b4c60dd..c7ed4204248 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -28,8 +28,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.MaxOutputTokens); Assert.Null(options.OutputModalities); Assert.Null(options.ToolMode); - Assert.Null(options.AIFunction); - Assert.Null(options.HostedMcpServerTool); Assert.Null(options.Tools); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 8b266605576..6111ee0fe8d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -617,15 +617,12 @@ public async Task AIFunction_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - var forcedFunction = AIFunctionFactory.Create((string query) => query, "SpecificSearch", "Search with specific parameters"); - using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model", - AIFunction = forcedFunction, - ToolMode = ChatToolMode.Auto, // Should be ignored when AIFunction is set + ToolMode = ChatToolMode.RequireSpecific("SpecificSearch"), }, GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), }; @@ -647,7 +644,7 @@ public async Task AIFunction_ForcedTool_Logged() #pragma warning disable MEAI001 // Type is for evaluation purposes only [Fact] - public async Task HostedMcpServerTool_ForcedTool_Logged() + public async Task RequireAny_ToolMode_Logged() { var sourceName = Guid.NewGuid().ToString(); var activities = new List(); @@ -656,15 +653,12 @@ public async Task HostedMcpServerTool_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - var mcpTool = new HostedMcpServerTool("github-server", "https://mcp.github.com"); - using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model", - HostedMcpServerTool = mcpTool, - ToolMode = ChatToolMode.Auto, // Should be ignored when HostedMcpServerTool is set + ToolMode = ChatToolMode.RequireAny, }, GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), }; @@ -681,7 +675,7 @@ public async Task HostedMcpServerTool_ForcedTool_Logged() } var activity = Assert.Single(activities); - Assert.Equal("mcp:github-server", activity.GetTagItem("gen_ai.request.tool_choice")); + Assert.Equal("required", activity.GetTagItem("gen_ai.request.tool_choice")); } #pragma warning restore MEAI001 From 4a6947e865275c114de9636977044bffc215244e Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:19:56 -0800 Subject: [PATCH 22/38] Improve XML docs for Usage on response and transcription messages --- ...altimeServerInputAudioTranscriptionMessage.cs | 6 +++++- .../RealtimeServerResponseCreatedMessage.cs | 7 ++++++- temp_check.cs | 16 ++++++++++++++++ temp_check.csx | 12 ++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 temp_check.cs create mode 100644 temp_check.csx diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs index 5ad03ab3997..ec011d3faee 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs @@ -43,8 +43,12 @@ public RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType ty public string? Transcription { get; set; } /// - /// Gets or sets the usage details for the transcription. + /// Gets or sets the transcription-specific usage, which is billed separately from the realtime model. /// + /// + /// This usage reflects the cost of the speech-to-text transcription and is billed according to the + /// ASR (Automatic Speech Recognition) model's pricing rather than the realtime model's pricing. + /// public UsageDetails? Usage { get; set; } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index ababa4a9947..1960f3b3d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -80,7 +80,12 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public ErrorContent? Error { get; set; } /// - /// Gets or sets the usage details for the response. + /// Gets or sets the per-response token usage for billing purposes. /// + /// + /// Populated when the response is complete (i.e., on ). + /// Input tokens include the entire conversation context, so they grow over successive turns + /// as previous output becomes input for later responses. + /// public UsageDetails? Usage { get; set; } } diff --git a/temp_check.cs b/temp_check.cs new file mode 100644 index 00000000000..6a7c6f60b04 --- /dev/null +++ b/temp_check.cs @@ -0,0 +1,16 @@ +using System; +using System.Reflection; +using System.Linq; + +class Program { + static void Main() { + var asm = Assembly.LoadFrom(@""Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll""); + var realtimeTypes = asm.GetTypes() + .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains(""Realtime"")) + .OrderBy(t => t.FullName); + foreach (var t in realtimeTypes) + { + Console.WriteLine(t.FullName); + } + } +} diff --git a/temp_check.csx b/temp_check.csx new file mode 100644 index 00000000000..3dc6853a57f --- /dev/null +++ b/temp_check.csx @@ -0,0 +1,12 @@ +using System; +using System.Reflection; +using System.Linq; + +var asm = Assembly.LoadFrom(@"Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll"); +var realtimeTypes = asm.GetTypes() + .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains("Realtime")) + .OrderBy(t => t.FullName); +foreach (var t in realtimeTypes) +{ + Console.WriteLine(t.FullName); +} From 29bbdca96e5a1ff7fd02fc76223d7f57bc198f9d Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:22:31 -0800 Subject: [PATCH 23/38] Remove leftover temp files --- temp_check.cs | 16 ---------------- temp_check.csx | 12 ------------ 2 files changed, 28 deletions(-) delete mode 100644 temp_check.cs delete mode 100644 temp_check.csx diff --git a/temp_check.cs b/temp_check.cs deleted file mode 100644 index 6a7c6f60b04..00000000000 --- a/temp_check.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Reflection; -using System.Linq; - -class Program { - static void Main() { - var asm = Assembly.LoadFrom(@""Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll""); - var realtimeTypes = asm.GetTypes() - .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains(""Realtime"")) - .OrderBy(t => t.FullName); - foreach (var t in realtimeTypes) - { - Console.WriteLine(t.FullName); - } - } -} diff --git a/temp_check.csx b/temp_check.csx deleted file mode 100644 index 3dc6853a57f..00000000000 --- a/temp_check.csx +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Reflection; -using System.Linq; - -var asm = Assembly.LoadFrom(@"Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll"); -var realtimeTypes = asm.GetTypes() - .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains("Realtime")) - .OrderBy(t => t.FullName); -foreach (var t in realtimeTypes) -{ - Console.WriteLine(t.FullName); -} From ab0e74d1d83db01114dfdc222360c86b5e3a5273 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:38:02 -0800 Subject: [PATCH 24/38] Improve ConversationId XML docs on RealtimeServerResponseCreatedMessage --- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 1960f3b3d02..dcac8956737 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -40,6 +40,11 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) /// /// Gets or sets the conversation ID associated with the response. /// + /// + /// Identifies which conversation within the session this response belongs to. + /// A session may have a default conversation to which items are automatically added, + /// or responses may be generated out-of-band (not associated with any conversation). + /// public string? ConversationId { get; set; } /// From 39a042d1a90fdca9a10bc36e10a6ee9180ca49a6 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 12:00:08 -0800 Subject: [PATCH 25/38] Split Text/Audio properties on RealtimeServerOutputTextAudioMessage Add separate Audio property for Base64-encoded audio data. Text is now only used for text and transcript content. Update OpenAI parser, OpenTelemetry session, and tests accordingly. --- .../RealtimeServerOutputTextAudioMessage.cs | 19 ++++++++++++++++--- .../OpenAIRealtimeSession.cs | 9 ++++++++- .../Realtime/OpenTelemetryRealtimeSession.cs | 2 +- .../Realtime/RealtimeServerMessageTests.cs | 1 + 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs index 7b1a5b3ea6d..be6b9137bfc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs @@ -32,14 +32,27 @@ public RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType type) public int? ContentIndex { get; set; } /// - /// Gets or sets the text or audio delta, or the final text or audio once the output is complete. + /// Gets or sets the text delta or final text content. /// /// - /// if dealing with audio content, this property may contain Base64-encoded audio data. - /// With , usually will have null Text value. + /// Populated for , , + /// , and messages. + /// For audio messages ( and ), + /// use instead. /// public string? Text { get; set; } + /// + /// Gets or sets the Base64-encoded audio data delta or final audio content. + /// + /// + /// Populated for messages. + /// For , this is typically + /// as the final audio is not included; use the accumulated deltas instead. + /// For text content, use instead. + /// + public string? Audio { get; set; } + /// /// Gets or sets the ID of the item containing the content part whose text has been updated. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 8266378e3fd..fa8ed439a5f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1664,7 +1664,14 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("delta", out var deltaElement)) { - msg.Text = deltaElement.GetString(); + if (serverMessageType is RealtimeServerMessageType.OutputAudioDelta) + { + msg.Audio = deltaElement.GetString(); + } + else + { + msg.Text = deltaElement.GetString(); + } } if (msg.Text is null && root.TryGetProperty("transcript", out deltaElement)) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index a559ce23973..b49e90cc4b5 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -451,7 +451,7 @@ private static void AddOutputMessagesTag(Activity? activity, List ("text", textAudioMsg.Text), RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone => - ("audio", string.IsNullOrEmpty(textAudioMsg.Text) ? "[audio data]" : textAudioMsg.Text), + ("audio", string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio), RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone => ("output_transcription", textAudioMsg.Text), _ => ("text", textAudioMsg.Text), diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 88eda1aac0c..06e1b155559 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -134,6 +134,7 @@ public void OutputTextAudioMessage_DefaultProperties() Assert.Null(message.ContentIndex); Assert.Null(message.Text); + Assert.Null(message.Audio); Assert.Null(message.ItemId); Assert.Null(message.OutputIndex); Assert.Null(message.ResponseId); From 46ae23dc4d79e085dbbcf88daa7be68d787208b9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 12:29:45 -0800 Subject: [PATCH 26/38] Replace RealtimeServerMessageType enum with readonly struct Follow the ChatRole smart-enum pattern: readonly struct with string Value, IEquatable, operators, and JsonConverter. Providers can now define custom message types by constructing new instances. Update pattern-matching in OpenTelemetryRealtimeSession to use == comparisons instead of constant patterns. --- .../Realtime/RealtimeServerMessageType.cs | 239 +++++++++--------- .../OpenAIRealtimeSession.cs | 2 +- .../Realtime/OpenTelemetryRealtimeSession.cs | 57 +++-- 3 files changed, 158 insertions(+), 140 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index 7f39ddb739e..7c722cf06a0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -1,165 +1,160 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.ComponentModel; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// -/// Represents the type of a real-time response. -/// This is used to identify the response type being received from the model. +/// Represents the type of a real-time server message. +/// This is used to identify the message type being received from the model. /// +/// +/// Well-known message types are provided as static properties. Providers may define additional +/// message types by constructing new instances with custom values. +/// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public enum RealtimeServerMessageType +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct RealtimeServerMessageType : IEquatable { - /// - /// Indicates that the response contains only raw content. - /// + /// Gets a message type indicating that the response contains only raw content. /// - /// This response type is to support extensibility for supporting custom content types not natively supported by the SDK. + /// This type supports extensibility for custom content types not natively supported by the SDK. /// - RawContentOnly, + public static RealtimeServerMessageType RawContentOnly { get; } = new("RawContentOnly"); - /// - /// Indicates the output of audio transcription for user audio written to the user audio buffer. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionCompleted, + /// Gets a message type indicating the output of audio transcription for user audio written to the user audio buffer. + public static RealtimeServerMessageType InputAudioTranscriptionCompleted { get; } = new("InputAudioTranscriptionCompleted"); - /// - /// Indicates the text value of an input audio transcription content part is updated with incremental transcription results. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionDelta, + /// Gets a message type indicating the text value of an input audio transcription content part is updated with incremental transcription results. + public static RealtimeServerMessageType InputAudioTranscriptionDelta { get; } = new("InputAudioTranscriptionDelta"); - /// - /// Indicates that the audio transcription for user audio written to the user audio buffer has failed. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionFailed, + /// Gets a message type indicating that the audio transcription for user audio written to the user audio buffer has failed. + public static RealtimeServerMessageType InputAudioTranscriptionFailed { get; } = new("InputAudioTranscriptionFailed"); - /// - /// Indicates the output text update with incremental results response. - /// - /// - /// The type is used with this response type. - /// - OutputTextDelta, + /// Gets a message type indicating the output text update with incremental results. + public static RealtimeServerMessageType OutputTextDelta { get; } = new("OutputTextDelta"); - /// - /// Indicates the output text is complete. - /// - /// - /// The type is used with this response type. - /// - OutputTextDone, + /// Gets a message type indicating the output text is complete. + public static RealtimeServerMessageType OutputTextDone { get; } = new("OutputTextDone"); - /// - /// Indicates the model-generated transcription of audio output updated. - /// - /// - /// The type is used with this response type. - /// - OutputAudioTranscriptionDelta, + /// Gets a message type indicating the model-generated transcription of audio output updated. + public static RealtimeServerMessageType OutputAudioTranscriptionDelta { get; } = new("OutputAudioTranscriptionDelta"); - /// - /// Indicates the model-generated transcription of audio output is done streaming. - /// - /// - /// The type is used with this response type. - /// - OutputAudioTranscriptionDone, + /// Gets a message type indicating the model-generated transcription of audio output is done streaming. + public static RealtimeServerMessageType OutputAudioTranscriptionDone { get; } = new("OutputAudioTranscriptionDone"); - /// - /// Indicates the audio output updated. - /// - /// - /// The type is used with this response type. - /// - OutputAudioDelta, + /// Gets a message type indicating the audio output updated. + public static RealtimeServerMessageType OutputAudioDelta { get; } = new("OutputAudioDelta"); - /// - /// Indicates the audio output is done streaming. - /// - /// - /// The type is used with this response type. - /// - OutputAudioDone, + /// Gets a message type indicating the audio output is done streaming. + public static RealtimeServerMessageType OutputAudioDone { get; } = new("OutputAudioDone"); - /// - /// Indicates the response has completed. - /// - /// - /// The type is used with this response type. - /// - ResponseDone, + /// Gets a message type indicating the response has completed. + public static RealtimeServerMessageType ResponseDone { get; } = new("ResponseDone"); - /// - /// Indicates the response has been created. - /// - /// - /// The type is used with this response type. - /// - ResponseCreated, + /// Gets a message type indicating the response has been created. + public static RealtimeServerMessageType ResponseCreated { get; } = new("ResponseCreated"); - /// - /// Indicates an individual output item in the response has completed. - /// - /// - /// The type is used with this response type. - /// - ResponseOutputItemDone, + /// Gets a message type indicating an individual output item in the response has completed. + public static RealtimeServerMessageType ResponseOutputItemDone { get; } = new("ResponseOutputItemDone"); - /// - /// Indicates an individual output item has been added to the response. - /// - /// - /// The type is used with this response type. - /// - ResponseOutputItemAdded, + /// Gets a message type indicating an individual output item has been added to the response. + public static RealtimeServerMessageType ResponseOutputItemAdded { get; } = new("ResponseOutputItemAdded"); - /// - /// Indicates an error occurred while processing the request. - /// - /// - /// The type is used with this response type. - /// - Error, + /// Gets a message type indicating an error occurred while processing the request. + public static RealtimeServerMessageType Error { get; } = new("Error"); - /// - /// Indicates that an MCP tool call is in progress. - /// - McpCallInProgress, + /// Gets a message type indicating that an MCP tool call is in progress. + public static RealtimeServerMessageType McpCallInProgress { get; } = new("McpCallInProgress"); - /// - /// Indicates that an MCP tool call has completed. - /// - McpCallCompleted, + /// Gets a message type indicating that an MCP tool call has completed. + public static RealtimeServerMessageType McpCallCompleted { get; } = new("McpCallCompleted"); + + /// Gets a message type indicating that an MCP tool call has failed. + public static RealtimeServerMessageType McpCallFailed { get; } = new("McpCallFailed"); + + /// Gets a message type indicating that listing MCP tools is in progress. + public static RealtimeServerMessageType McpListToolsInProgress { get; } = new("McpListToolsInProgress"); + + /// Gets a message type indicating that listing MCP tools has completed. + public static RealtimeServerMessageType McpListToolsCompleted { get; } = new("McpListToolsCompleted"); + + /// Gets a message type indicating that listing MCP tools has failed. + public static RealtimeServerMessageType McpListToolsFailed { get; } = new("McpListToolsFailed"); /// - /// Indicates that an MCP tool call has failed. + /// Gets the value associated with this . /// - McpCallFailed, + public string Value { get; } /// - /// Indicates that listing MCP tools is in progress. + /// Initializes a new instance of the struct with the provided value. /// - McpListToolsInProgress, + /// The value to associate with this . + [JsonConstructor] + public RealtimeServerMessageType(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } /// - /// Indicates that listing MCP tools has completed. + /// Returns a value indicating whether two instances are equivalent, as determined by a + /// case-insensitive comparison of their values. /// - McpListToolsCompleted, + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(RealtimeServerMessageType left, RealtimeServerMessageType right) + { + return left.Equals(right); + } /// - /// Indicates that listing MCP tools has failed. + /// Returns a value indicating whether two instances are not equivalent, as determined by a + /// case-insensitive comparison of their values. /// - McpListToolsFailed, + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(RealtimeServerMessageType left, RealtimeServerMessageType right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is RealtimeServerMessageType other && Equals(other); + + /// + public bool Equals(RealtimeServerMessageType other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => Value is null ? 0 : StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value ?? string.Empty; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override RealtimeServerMessageType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, RealtimeServerMessageType value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index fa8ed439a5f..5da44e4138c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1664,7 +1664,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("delta", out var deltaElement)) { - if (serverMessageType is RealtimeServerMessageType.OutputAudioDelta) + if (serverMessageType == RealtimeServerMessageType.OutputAudioDelta) { msg.Audio = deltaElement.GetString(); } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index b49e90cc4b5..7f4057b3894 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -364,14 +364,30 @@ private static void AddOutputMessagesTag(Activity? activity, ListGets the output modality from a server message, if applicable. private static string? GetOutputModality(RealtimeServerMessage message) { - return message switch + if (message is RealtimeServerOutputTextAudioMessage textAudio) { - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputTextDelta or RealtimeServerMessageType.OutputTextDone } => "text", - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone } => "audio", - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone } => "transcription", - RealtimeServerResponseOutputItemMessage => "item", - _ => null, - }; + if (textAudio.Type == RealtimeServerMessageType.OutputTextDelta || textAudio.Type == RealtimeServerMessageType.OutputTextDone) + { + return "text"; + } + + if (textAudio.Type == RealtimeServerMessageType.OutputAudioDelta || textAudio.Type == RealtimeServerMessageType.OutputAudioDone) + { + return "audio"; + } + + if (textAudio.Type == RealtimeServerMessageType.OutputAudioTranscriptionDelta || textAudio.Type == RealtimeServerMessageType.OutputAudioTranscriptionDone) + { + return "transcription"; + } + } + + if (message is RealtimeServerResponseOutputItemMessage) + { + return "item"; + } + + return null; } /// Extracts an OTel message from a realtime client message. @@ -445,17 +461,24 @@ private static void AddOutputMessagesTag(Activity? activity, List - ("text", textAudioMsg.Text), - RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone => - ("audio", string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio), - RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone => - ("output_transcription", textAudioMsg.Text), - _ => ("text", textAudioMsg.Text), - }; + partType = "audio"; + content = string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio; + } + else if (textAudioMsg.Type == RealtimeServerMessageType.OutputAudioTranscriptionDelta || textAudioMsg.Type == RealtimeServerMessageType.OutputAudioTranscriptionDone) + { + partType = "output_transcription"; + content = textAudioMsg.Text; + } + else + { + partType = "text"; + content = textAudioMsg.Text; + } // Skip if no meaningful content if (string.IsNullOrEmpty(content)) From 156b3764a3d3c7210e4395cb81e98e1c352a94bc Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 13:40:44 -0800 Subject: [PATCH 27/38] Move Parameter into ErrorContent.Details on error messages Remove the Parameter property from RealtimeServerErrorMessage and map error.param to ErrorContent.Details instead. Improve ErrorEventId XML docs to clarify it correlates to the originating client event. --- .../Realtime/RealtimeServerErrorMessage.cs | 11 +++++------ .../OpenAIRealtimeSession.cs | 2 +- .../Realtime/RealtimeServerMessageTests.cs | 6 ++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index 85ce29d73a4..e785318e9c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -29,13 +29,12 @@ public RealtimeServerErrorMessage() public ErrorContent? Error { get; set; } /// - /// Gets or sets an optional event ID caused the error. + /// Gets or sets the event ID of the client event that caused the error. /// + /// + /// This is specific to event-driven protocols where multiple client events may be in-flight, + /// allowing correlation of the error to the originating client request. + /// public string? ErrorEventId { get; set; } - /// - /// Gets or sets an optional parameter providing additional context about the error. - /// - public string? Parameter { get; set; } - } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 5da44e4138c..2dca1fa54c6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1555,7 +1555,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (errorElement.TryGetProperty("param", out var paramElement)) { - msg.Parameter = paramElement.GetString(); + msg.Error.Details = paramElement.GetString(); } return msg; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 06e1b155559..8d327927b0e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -51,24 +51,22 @@ public void ErrorMessage_DefaultProperties() Assert.Null(message.Error); Assert.Null(message.ErrorEventId); - Assert.Null(message.Parameter); } [Fact] public void ErrorMessage_Properties_Roundtrip() { - var error = new ErrorContent("Test error"); + var error = new ErrorContent("Test error") { Details = "temperature" }; var message = new RealtimeServerErrorMessage { Error = error, ErrorEventId = "evt_bad", - Parameter = "temperature", EventId = "evt_err_1", }; Assert.Same(error, message.Error); Assert.Equal("evt_bad", message.ErrorEventId); - Assert.Equal("temperature", message.Parameter); + Assert.Equal("temperature", message.Error.Details); Assert.Equal("evt_err_1", message.EventId); Assert.IsAssignableFrom(message); } From f59afd691c6912565262fc46e05c378fb872c408 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 13:49:17 -0800 Subject: [PATCH 28/38] Add RawRepresentation property to RealtimeContentItem Add object? RawRepresentation to hold the original provider data structure, following the same pattern as other types in the abstraction layer (e.g., ChatMessage). Updated tests accordingly. --- .../Realtime/RealtimeContentItem.cs | 6 ++++++ .../Realtime/RealtimeContentItemTests.cs | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs index 1aaf4e0721f..9f3040dc326 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -52,4 +52,10 @@ public RealtimeContentItem(IList contents, string? id = null, ChatRol /// Gets or sets the content of the conversation item. /// public IList Contents { get; set; } + + /// + /// Gets or sets the raw representation of the conversation item. + /// This can be used to hold the original data structure received from or sent to the provider. + /// + public object? RawRepresentation { get; set; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs index b27a42c2bc8..c2e50936894 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs @@ -19,6 +19,7 @@ public void Constructor_WithContentsOnly_PropsDefaulted() Assert.Same(contents, item.Contents); Assert.Null(item.Id); Assert.Null(item.Role); + Assert.Null(item.RawRepresentation); } [Fact] @@ -42,10 +43,12 @@ public void Properties_Roundtrip() item.Id = "new_id"; item.Role = ChatRole.Assistant; item.Contents = newContents; + item.RawRepresentation = "raw_data"; Assert.Equal("new_id", item.Id); Assert.Equal(ChatRole.Assistant, item.Role); Assert.Same(newContents, item.Contents); + Assert.Equal("raw_data", item.RawRepresentation); } [Fact] From e8393c144ee369ea0a7faf8ba00b48cbcb815fab Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 14:03:49 -0800 Subject: [PATCH 29/38] Rename Metadata to AdditionalProperties for consistency Rename the Metadata property to AdditionalProperties on both RealtimeClientResponseCreateMessage and RealtimeServerResponseCreatedMessage to be consistent with the established pattern used across the AI abstractions (ChatMessage, ChatOptions, AIContent, etc.). Updated XML docs, OpenAI provider, OTel session, and tests accordingly. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 9 +++++++-- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 9 +++++++-- .../OpenAIRealtimeSession.cs | 6 +++--- .../Realtime/OpenTelemetryRealtimeSession.cs | 2 +- .../Realtime/RealtimeClientMessageTests.cs | 6 +++--- .../Realtime/RealtimeServerMessageTests.cs | 6 +++--- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index a840195b4fd..5b6c8703018 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -52,9 +52,14 @@ public RealtimeClientResponseCreateMessage() public int? MaxOutputTokens { get; set; } /// - /// Gets or sets additional metadata for the message. + /// Gets or sets any additional properties associated with the response request. /// - public AdditionalPropertiesDictionary? Metadata { get; set; } + /// + /// This can be used to attach arbitrary key-value metadata to a response request + /// for tracking or disambiguation purposes (e.g., correlating multiple simultaneous responses). + /// Providers may map this to their own metadata fields. + /// + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// /// Gets or sets the output modalities for the response. like "text", "audio". diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index dcac8956737..09d62b582e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -59,9 +59,14 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public int? MaxOutputTokens { get; set; } /// - /// Gets or sets additional metadata for the message. + /// Gets or sets any additional properties associated with the response. /// - public AdditionalPropertiesDictionary? Metadata { get; set; } + /// + /// Contains arbitrary key-value metadata attached to the response. + /// This is the metadata that was provided when the response was created + /// (e.g., for tracking or disambiguating multiple simultaneous responses). + /// + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// /// Gets or sets the list of the conversation items included in the response. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 2dca1fa54c6..f6257930f46 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -384,10 +384,10 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel responseObj["max_output_tokens"] = responseCreate.MaxOutputTokens.Value; } - if (responseCreate.Metadata is { Count: > 0 }) + if (responseCreate.AdditionalProperties is { Count: > 0 }) { var metadataObj = new JsonObject(); - foreach (var kvp in responseCreate.Metadata) + foreach (var kvp in responseCreate.AdditionalProperties) { metadataObj[kvp.Key] = JsonValue.Create(kvp.Value); } @@ -1779,7 +1779,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess : JsonSerializer.Deserialize(property.Value.GetRawText()); } - msg.Metadata = metadataDict; + msg.AdditionalProperties = metadataDict; } if (responseElement.TryGetProperty("output_modalities", out var outputModalitiesElement)) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 7f4057b3894..bcb76facb5a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -964,7 +964,7 @@ private void TraceStreamingResponse( if (response is not null && activity is not null) { // Log metadata first so standard tags take precedence if keys collide - if (EnableSensitiveData && response.Metadata is { } metadata) + if (EnableSensitiveData && response.AdditionalProperties is { } metadata) { foreach (var prop in metadata) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index a2d0530d39c..cc67e8347ea 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -137,7 +137,7 @@ public void ResponseCreateMessage_DefaultProperties() Assert.False(message.ExcludeFromConversation); Assert.Null(message.Instructions); Assert.Null(message.MaxOutputTokens); - Assert.Null(message.Metadata); + Assert.Null(message.AdditionalProperties); Assert.Null(message.OutputModalities); Assert.Null(message.ToolMode); Assert.Null(message.Tools); @@ -163,7 +163,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() message.ExcludeFromConversation = true; message.Instructions = "Be brief"; message.MaxOutputTokens = 100; - message.Metadata = metadata; + message.AdditionalProperties = metadata; message.OutputModalities = modalities; message.ToolMode = ChatToolMode.Auto; message.Tools = tools; @@ -174,7 +174,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() Assert.True(message.ExcludeFromConversation); Assert.Equal("Be brief", message.Instructions); Assert.Equal(100, message.MaxOutputTokens); - Assert.Same(metadata, message.Metadata); + Assert.Same(metadata, message.AdditionalProperties); Assert.Same(modalities, message.OutputModalities); Assert.Equal(ChatToolMode.Auto, message.ToolMode); Assert.Same(tools, message.Tools); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 8d327927b0e..15308933201 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -177,7 +177,7 @@ public void ResponseCreatedMessage_DefaultProperties() Assert.Null(message.ConversationId); Assert.Null(message.ResponseId); Assert.Null(message.MaxOutputTokens); - Assert.Null(message.Metadata); + Assert.Null(message.AdditionalProperties); Assert.Null(message.Items); Assert.Null(message.OutputModalities); Assert.Null(message.Status); @@ -205,7 +205,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() ConversationId = "conv_1", ResponseId = "resp_1", MaxOutputTokens = 1000, - Metadata = metadata, + AdditionalProperties = metadata, Items = items, OutputModalities = modalities, Status = "completed", @@ -218,7 +218,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() Assert.Equal("conv_1", message.ConversationId); Assert.Equal("resp_1", message.ResponseId); Assert.Equal(1000, message.MaxOutputTokens); - Assert.Same(metadata, message.Metadata); + Assert.Same(metadata, message.AdditionalProperties); Assert.Same(items, message.Items); Assert.Same(modalities, message.OutputModalities); Assert.Equal("completed", message.Status); From 6aae7d3d44962fa08e6f0b414fe479168abff637 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 14:11:49 -0800 Subject: [PATCH 30/38] Improve MaxOutputTokens XML docs to clarify modality scope Clarify that MaxOutputTokens is a total budget across all output modalities (text, audio) and tool calls, not per-modality. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 6 +++++- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 5b6c8703018..b626ac950c9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -47,8 +47,12 @@ public RealtimeClientResponseCreateMessage() public string? Instructions { get; set; } /// - /// Gets or sets the maximum number of output tokens for the response. + /// Gets or sets the maximum number of output tokens for the response, inclusive of all modalities and tool calls. /// + /// + /// This limit applies to the total output tokens regardless of modality (text, audio, etc.). + /// If , the provider's default limit is used. + /// public int? MaxOutputTokens { get; set; } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 09d62b582e0..36823ae99e8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -53,9 +53,12 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public string? ResponseId { get; set; } /// - /// Gets or sets the maximum number of output tokens for the response. - /// If 0, the service will apply its own limit. + /// Gets or sets the maximum number of output tokens for the response, inclusive of all modalities and tool calls. /// + /// + /// This limit applies to the total output tokens regardless of modality (text, audio, etc.). + /// If , the provider's default limit was used. + /// public int? MaxOutputTokens { get; set; } /// From c7f041b455baec35a2fff68b3881a57ae8f75522 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:08:14 -0800 Subject: [PATCH 31/38] Improve XML docs on RealtimeClientResponseCreateMessage properties Clarify that ExcludeFromConversation creates an out-of-band response whose output is not added to conversation history. Document that Instructions, Tools, ToolMode, OutputModalities, OutputAudioOptions, and OutputVoice are per-response overrides of session configuration. --- .../RealtimeClientResponseCreateMessage.cs | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index b626ac950c9..2339ecb8f84 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -26,24 +26,41 @@ public RealtimeClientResponseCreateMessage() public IList? Items { get; set; } /// - /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// Gets or sets the output audio options for the response. /// + /// + /// If set, overrides the session-level audio output configuration for this response only. + /// If , the session's default audio options are used. + /// public RealtimeAudioFormat? OutputAudioOptions { get; set; } /// /// Gets or sets the voice of the output audio. /// + /// + /// If set, overrides the session-level voice for this response only. + /// If , the session's default voice is used. + /// public string? OutputVoice { get; set; } /// - /// Gets or sets a value indicating whether the response should be excluded from the conversation history. + /// Gets or sets a value indicating whether the response output should be excluded from the conversation context. /// + /// + /// When , the response is generated out-of-band: the model produces output + /// but the resulting items are not added to the conversation history, so they will not appear + /// as context for subsequent responses. Defaults to , meaning response + /// output is added to the default conversation. + /// public bool ExcludeFromConversation { get; set; } /// - /// Gets or sets the instructions allows the client to guide the model on desired responses. - /// If null, the default conversation instructions will be used. + /// Gets or sets the instructions that guide the model on desired responses. /// + /// + /// If set, overrides the session-level instructions for this response only. + /// If , the session's default instructions are used. + /// public string? Instructions { get; set; } /// @@ -66,18 +83,29 @@ public RealtimeClientResponseCreateMessage() public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// - /// Gets or sets the output modalities for the response. like "text", "audio". - /// If null, then default conversation modalities will be used. + /// Gets or sets the output modalities for the response (e.g., "text", "audio"). /// + /// + /// If set, overrides the session-level output modalities for this response only. + /// If , the session's default modalities are used. + /// public IList? OutputModalities { get; set; } /// /// Gets or sets the tool choice mode for the response. /// + /// + /// If set, overrides the session-level tool choice for this response only. + /// If , the session's default tool choice is used. + /// public ChatToolMode? ToolMode { get; set; } /// /// Gets or sets the AI tools available for generating the response. /// + /// + /// If set, overrides the session-level tools for this response only. + /// If , the session's default tools are used. + /// public IList? Tools { get; set; } } From cde880a6cf367bd951847179f434ed301ba86d59 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:18:49 -0800 Subject: [PATCH 32/38] Improve RealtimeClientResponseCreateMessage class-level XML doc Clarify that this message triggers model inference and that its properties are per-response overrides of session configuration. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 2339ecb8f84..bcd4a016f97 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -8,8 +8,14 @@ namespace Microsoft.Extensions.AI; /// -/// Represents a real-time message for creating a response item. +/// Represents a client message that triggers model inference to generate a response. /// +/// +/// Sending this message instructs the provider to generate a new response from the model. +/// The response may include one or more output items (text, audio, or tool calls). +/// Properties on this message optionally override the session-level configuration +/// for this response only. +/// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class RealtimeClientResponseCreateMessage : RealtimeClientMessage { From d2516c224713953f77f5c5029b3a02d7236a656f Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:33:49 -0800 Subject: [PATCH 33/38] Rename EventId to MessageId for terminology consistency Rename EventId to MessageId on RealtimeClientMessage and RealtimeServerMessage, and ErrorEventId to ErrorMessageId on RealtimeServerErrorMessage. The abstraction uses 'message' terminology throughout (class names, docs, method signatures), so properties should match. The OpenAI provider maps MessageId to/from the wire protocol's event_id field. --- .../Realtime/RealtimeClientMessage.cs | 4 +-- .../Realtime/RealtimeServerErrorMessage.cs | 6 ++-- .../Realtime/RealtimeServerMessage.cs | 4 +-- .../OpenAIRealtimeSession.cs | 28 +++++++++---------- .../Realtime/LoggingRealtimeSession.cs | 8 +++--- .../Realtime/RealtimeClientMessageTests.cs | 20 ++++++------- .../Realtime/RealtimeServerMessageTests.cs | 16 +++++------ .../DelegatingRealtimeSessionTests.cs | 4 +-- .../FunctionInvokingRealtimeSessionTests.cs | 12 ++++---- .../Realtime/LoggingRealtimeSessionTests.cs | 10 +++---- .../OpenTelemetryRealtimeSessionTests.cs | 2 +- .../Realtime/RealtimeSessionBuilderTests.cs | 4 +-- 12 files changed, 59 insertions(+), 59 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs index da4336d2314..0f035933462 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs @@ -13,10 +13,10 @@ namespace Microsoft.Extensions.AI; public class RealtimeClientMessage { /// - /// Gets or sets the optional event ID associated with the message. + /// Gets or sets the optional message ID associated with the message. /// This can be used for tracking and correlation purposes. /// - public string? EventId { get; set; } + public string? MessageId { get; set; } /// /// Gets or sets the raw representation of the message. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index e785318e9c8..ea3131482fa 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -29,12 +29,12 @@ public RealtimeServerErrorMessage() public ErrorContent? Error { get; set; } /// - /// Gets or sets the event ID of the client event that caused the error. + /// Gets or sets the message ID of the client message that caused the error. /// /// - /// This is specific to event-driven protocols where multiple client events may be in-flight, + /// This is specific to event-driven protocols where multiple client messages may be in-flight, /// allowing correlation of the error to the originating client request. /// - public string? ErrorEventId { get; set; } + public string? ErrorMessageId { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs index d2af4b4a99a..0e023fde4f4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs @@ -18,10 +18,10 @@ public class RealtimeServerMessage public RealtimeServerMessageType Type { get; set; } /// - /// Gets or sets the optional event ID associated with the response. + /// Gets or sets the optional message ID associated with the response. /// This can be used for tracking and correlation purposes. /// - public string? EventId { get; set; } + public string? MessageId { get; set; } /// /// Gets or sets the raw representation of the response. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index f6257930f46..499d747bd7b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -285,9 +285,9 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel JsonObject? jsonMessage = new JsonObject(); - if (message.EventId is not null) + if (message.MessageId is not null) { - jsonMessage["event_id"] = message.EventId; + jsonMessage["event_id"] = message.MessageId; } switch (message) @@ -554,11 +554,11 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel jsonMessage = rawJsonObject; } - // Preserve EventId if it was set on the message but not in the raw representation. - if (jsonMessage is not null && message.EventId is not null && + // Preserve MessageId if it was set on the message but not in the raw representation. + if (jsonMessage is not null && message.MessageId is not null && !jsonMessage.ContainsKey("event_id")) { - jsonMessage["event_id"] = message.EventId; + jsonMessage["event_id"] = message.MessageId; } break; @@ -1550,7 +1550,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (errorElement.TryGetProperty("param", out var paramElement)) @@ -1575,7 +1575,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("content_index", out var contentIndexElement)) @@ -1639,7 +1639,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1695,7 +1695,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1734,7 +1734,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (responseElement.TryGetProperty("audio", out var responseAudioElement) && @@ -1845,7 +1845,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1884,7 +1884,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType) { RawRepresentation = root.Clone(), - EventId = root.TryGetProperty("event_id", out var eventIdElement) ? eventIdElement.GetString() : null, + MessageId = root.TryGetProperty("event_id", out var eventIdElement) ? eventIdElement.GetString() : null, }; string? itemId = root.TryGetProperty("item_id", out var itemIdElement) ? itemIdElement.GetString() : null; @@ -1949,7 +1949,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.RawContentOnly) { RawRepresentation = root.Clone(), - EventId = root.TryGetProperty("event_id", out var evtElement) ? evtElement.GetString() : null, + MessageId = root.TryGetProperty("event_id", out var evtElement) ? evtElement.GetString() : null, }; } @@ -1961,7 +1961,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } return msg; diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index a61ed689ec3..520b0748b09 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -217,9 +217,9 @@ private string GetLoggableString(RealtimeClientMessage message) { obj["content"] = AsJson(message.RawRepresentation); } - else if (message.EventId is not null) + else if (message.MessageId is not null) { - obj["eventId"] = message.EventId; + obj["messageId"] = message.MessageId; } return obj.ToJsonString(); @@ -240,9 +240,9 @@ private string GetLoggableString(RealtimeServerMessage message) { obj["content"] = AsJson(message.RawRepresentation); } - else if (message.EventId is not null) + else if (message.MessageId is not null) { - obj["eventId"] = message.EventId; + obj["messageId"] = message.MessageId; } return obj.ToJsonString(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index cc67e8347ea..f97a521f083 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -16,7 +16,7 @@ public void RealtimeClientMessage_DefaultProperties() { var message = new RealtimeClientMessage(); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); Assert.Null(message.RawRepresentation); } @@ -26,11 +26,11 @@ public void RealtimeClientMessage_Properties_Roundtrip() var rawObj = new object(); var message = new RealtimeClientMessage { - EventId = "evt_001", + MessageId = "evt_001", RawRepresentation = rawObj, }; - Assert.Equal("evt_001", message.EventId); + Assert.Equal("evt_001", message.MessageId); Assert.Same(rawObj, message.RawRepresentation); } @@ -76,10 +76,10 @@ public void ConversationItemCreateMessage_InheritsClientMessage() var item = new RealtimeContentItem([new TextContent("Hello")]); var message = new RealtimeClientConversationItemCreateMessage(item) { - EventId = "evt_create_1", + MessageId = "evt_create_1", }; - Assert.Equal("evt_create_1", message.EventId); + Assert.Equal("evt_create_1", message.MessageId); Assert.IsAssignableFrom(message); } @@ -110,10 +110,10 @@ public void InputAudioBufferAppendMessage_InheritsClientMessage() var audioContent = new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"); var message = new RealtimeClientInputAudioBufferAppendMessage(audioContent) { - EventId = "evt_append_1", + MessageId = "evt_append_1", }; - Assert.Equal("evt_append_1", message.EventId); + Assert.Equal("evt_append_1", message.MessageId); Assert.IsAssignableFrom(message); } @@ -123,7 +123,7 @@ public void InputAudioBufferCommitMessage_Constructor() var message = new RealtimeClientInputAudioBufferCommitMessage(); Assert.IsAssignableFrom(message); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); } [Fact] @@ -185,11 +185,11 @@ public void ResponseCreateMessage_InheritsClientMessage() { var message = new RealtimeClientResponseCreateMessage { - EventId = "evt_resp_1", + MessageId = "evt_resp_1", RawRepresentation = "raw", }; - Assert.Equal("evt_resp_1", message.EventId); + Assert.Equal("evt_resp_1", message.MessageId); Assert.Equal("raw", message.RawRepresentation); Assert.IsAssignableFrom(message); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 15308933201..fe1c1ef9837 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -16,7 +16,7 @@ public void RealtimeServerMessage_DefaultProperties() var message = new RealtimeServerMessage(); Assert.Equal(default, message.Type); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); Assert.Null(message.RawRepresentation); } @@ -27,12 +27,12 @@ public void RealtimeServerMessage_Properties_Roundtrip() var message = new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, - EventId = "evt_001", + MessageId = "evt_001", RawRepresentation = rawObj, }; Assert.Equal(RealtimeServerMessageType.ResponseDone, message.Type); - Assert.Equal("evt_001", message.EventId); + Assert.Equal("evt_001", message.MessageId); Assert.Same(rawObj, message.RawRepresentation); } @@ -50,7 +50,7 @@ public void ErrorMessage_DefaultProperties() var message = new RealtimeServerErrorMessage(); Assert.Null(message.Error); - Assert.Null(message.ErrorEventId); + Assert.Null(message.ErrorMessageId); } [Fact] @@ -60,14 +60,14 @@ public void ErrorMessage_Properties_Roundtrip() var message = new RealtimeServerErrorMessage { Error = error, - ErrorEventId = "evt_bad", - EventId = "evt_err_1", + ErrorMessageId = "evt_bad", + MessageId = "evt_err_1", }; Assert.Same(error, message.Error); - Assert.Equal("evt_bad", message.ErrorEventId); + Assert.Equal("evt_bad", message.ErrorMessageId); Assert.Equal("temperature", message.Error.Details); - Assert.Equal("evt_err_1", message.EventId); + Assert.Equal("evt_err_1", message.MessageId); Assert.IsAssignableFrom(message); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index 617fe1f0fb9..a9f6df9b410 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -54,7 +54,7 @@ public async Task UpdateAsync_DelegatesToInner() public async Task InjectClientMessageAsync_DelegatesToInner() { var called = false; - var sentMessage = new RealtimeClientMessage { EventId = "evt_001" }; + var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; using var inner = new TestRealtimeSession { InjectClientMessageAsyncCallback = (msg, _) => @@ -73,7 +73,7 @@ public async Task InjectClientMessageAsync_DelegatesToInner() [Fact] public async Task GetStreamingResponseAsync_DelegatesToInner() { - var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, EventId = "evt_002" }; + var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(expected, ct), diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index 554349ad4c8..fb953217bc3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -81,8 +81,8 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() { var serverMessages = new RealtimeServerMessage[] { - new() { Type = RealtimeServerMessageType.ResponseCreated, EventId = "evt_001" }, - new() { Type = RealtimeServerMessageType.ResponseDone, EventId = "evt_002" }, + new() { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }, + new() { Type = RealtimeServerMessageType.ResponseDone, MessageId = "evt_002" }, }; using var inner = new TestRealtimeSession @@ -98,8 +98,8 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() } Assert.Equal(2, received.Count); - Assert.Equal("evt_001", received[0].EventId); - Assert.Equal("evt_002", received[1].EventId); + Assert.Equal("evt_001", received[0].MessageId); + Assert.Equal("evt_002", received[1].MessageId); } [Fact] @@ -395,7 +395,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), - new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, EventId = "should_not_reach" }, + new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), InjectClientMessageAsyncCallback = (msg, _) => { @@ -621,7 +621,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_decl", "my_declaration", null), - new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, EventId = "should_not_reach" }, + new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), InjectClientMessageAsyncCallback = (msg, _) => { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 52672dd7689..5a26dc339d8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -108,7 +108,7 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel .UseLogging() .Build(services); - await session.InjectClientMessageAsync(new RealtimeClientMessage { EventId = "test-event-123" }); + await session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) @@ -146,8 +146,8 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) static async IAsyncEnumerable GetMessagesAsync() { await Task.Yield(); - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputTextDelta, EventId = "event-1" }; - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, EventId = "event-2" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputTextDelta, MessageId = "event-1" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, MessageId = "event-2" }; } using var session = innerSession @@ -360,7 +360,7 @@ public async Task InjectClientMessageAsync_LogsCancellation() cts.Cancel(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage { EventId = "evt_cancel" }, cts.Token)); + session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, @@ -511,6 +511,6 @@ private static async IAsyncEnumerable GetClientMessages( { _ = cancellationToken; await Task.CompletedTask.ConfigureAwait(false); - yield return new RealtimeClientMessage { EventId = "client_evt_1" }; + yield return new RealtimeClientMessage { MessageId = "client_evt_1" }; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 6111ee0fe8d..79552329be4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -58,7 +58,7 @@ static async IAsyncEnumerable CallbackAsync( // Just consume the update } - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, EventId = "evt_001" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = " there!" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) { OutputIndex = 0, Text = "Hello there!" }; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index 997d28aefcf..25b10374d75 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -142,7 +142,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() var intercepted = false; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { EventId = "inner" }, ct), + GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; var builder = new RealtimeSessionBuilder(inner); @@ -155,7 +155,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() using var pipeline = builder.Build(); await foreach (var msg in pipeline.GetStreamingResponseAsync(EmptyUpdates())) { - Assert.Equal("inner", msg.EventId); + Assert.Equal("inner", msg.MessageId); } Assert.True(intercepted); From 1d15400439c1b2e222e49125480988e3b76ddc4c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:46:35 -0800 Subject: [PATCH 34/38] Remove blank line between using directives in audio buffer messages --- .../Realtime/RealtimeClientInputAudioBufferAppendMessage.cs | 1 - .../Realtime/RealtimeClientInputAudioBufferCommitMessage.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index 86c54816a89..2e1f9c998d2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; - using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs index 8415588525f..15be87316d3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; - using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; From 0ca0a172b0807b39aa159f5d130bc652e82d12b0 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 16:02:52 -0800 Subject: [PATCH 35/38] Rename RealtimeAudioFormat.Type to MediaType for consistency Rename to match the established MediaType naming convention used across the abstractions (DataContent, HostedFileContent, UriContent, ImageGenerationOptions). Updated OpenAI provider and tests. --- .../Realtime/RealtimeAudioFormat.cs | 8 ++++---- .../OpenAIRealtimeSession.cs | 10 +++++----- .../Realtime/RealtimeAudioFormatTests.cs | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs index 37535d634d2..93c97d08fd2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -15,16 +15,16 @@ public class RealtimeAudioFormat /// /// Initializes a new instance of the class. /// - public RealtimeAudioFormat(string type, int sampleRate) + public RealtimeAudioFormat(string mediaType, int sampleRate) { - Type = type; + MediaType = mediaType; SampleRate = sampleRate; } /// - /// Gets or sets the type of audio. For example, "audio/pcm". + /// Gets or sets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). /// - public string Type { get; set; } + public string MediaType { get; set; } /// /// Gets or sets the sample rate of the audio in Hertz. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 499d747bd7b..ffb1cd88a30 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -140,7 +140,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { var audioInputFormatElement = new JsonObject { - ["type"] = options.InputAudioFormat.Type, + ["type"] = options.InputAudioFormat.MediaType, }; if (options.InputAudioFormat.SampleRate.HasValue) { @@ -206,7 +206,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { var audioOutputFormatElement = new JsonObject { - ["type"] = options.OutputAudioFormat.Type, + ["type"] = options.OutputAudioFormat.MediaType, }; if (options.OutputAudioFormat.SampleRate.HasValue) { @@ -303,12 +303,12 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel var outputObj = new JsonObject(); var formatObj = new JsonObject(); - switch (responseCreate.OutputAudioOptions.Type) + switch (responseCreate.OutputAudioOptions.MediaType) { case "audio/pcm": if (responseCreate.OutputAudioOptions.SampleRate == 24000) { - formatObj["type"] = responseCreate.OutputAudioOptions.Type; + formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; formatObj["rate"] = 24000; } @@ -316,7 +316,7 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel case "audio/pcmu": case "audio/pcma": - formatObj["type"] = responseCreate.OutputAudioOptions.Type; + formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; break; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs index 181bb5a64ed..e6af6ba6b97 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs @@ -14,7 +14,7 @@ public void Constructor_SetsProperties() { var format = new RealtimeAudioFormat("audio/pcm", 16000); - Assert.Equal("audio/pcm", format.Type); + Assert.Equal("audio/pcm", format.MediaType); Assert.Equal(16000, format.SampleRate); } @@ -23,11 +23,11 @@ public void Properties_Roundtrip() { var format = new RealtimeAudioFormat("audio/pcm", 16000) { - Type = "audio/wav", + MediaType = "audio/wav", SampleRate = 24000, }; - Assert.Equal("audio/wav", format.Type); + Assert.Equal("audio/wav", format.MediaType); Assert.Equal(24000, format.SampleRate); } From 03014ce34d95c17ed4e0ff5e80ec05f646303f04 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 11:02:39 -0800 Subject: [PATCH 36/38] Refactor IRealtimeSession: rename InjectClientMessageAsync to SendClientMessageAsync and remove updates parameter from GetStreamingResponseAsync - Rename InjectClientMessageAsync -> SendClientMessageAsync across all implementations - Remove IAsyncEnumerable updates parameter from GetStreamingResponseAsync - Move per-message telemetry from WrapClientMessagesForTelemetryAsync into SendClientMessageAsync override in OpenTelemetryRealtimeSession - Delete WrapUpdatesWithLoggingAsync from LoggingRealtimeSession - Delete WrapClientMessagesForTelemetryAsync from OpenTelemetryRealtimeSession - Update AnonymousDelegatingRealtimeSession delegate signature - Update RealtimeSessionBuilder.Use overload signature - Update all tests to use new API Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Realtime/DelegatingRealtimeSession.cs | 8 +- .../Realtime/IRealtimeSession.cs | 15 +- .../OpenAIRealtimeSession.cs | 30 +- .../AnonymousDelegatingRealtimeSession.cs | 10 +- .../FunctionInvokingRealtimeSession.cs | 8 +- .../Realtime/LoggingRealtimeSession.cs | 56 +--- .../Realtime/OpenTelemetryRealtimeSession.cs | 78 ++--- .../Realtime/RealtimeSessionBuilder.cs | 6 +- .../TestRealtimeSession.cs | 14 +- .../OpenAIRealtimeSessionTests.cs | 22 +- .../DelegatingRealtimeSessionTests.cs | 28 +- .../FunctionInvokingRealtimeSessionTests.cs | 99 +++--- .../Realtime/LoggingRealtimeSessionTests.cs | 117 ++----- .../OpenTelemetryRealtimeSessionTests.cs | 313 ++++++++++-------- .../Realtime/RealtimeSessionBuilderTests.cs | 21 +- 15 files changed, 327 insertions(+), 498 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs index e9476b2aefe..4bb1defb3b9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -68,8 +68,8 @@ protected virtual async ValueTask DisposeAsyncCore() public virtual RealtimeSessionOptions? Options => InnerSession.Options; /// - public virtual Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => - InnerSession.InjectClientMessageAsync(message, cancellationToken); + public virtual Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + InnerSession.SendClientMessageAsync(message, cancellationToken); /// public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => @@ -77,8 +77,8 @@ public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToke /// public virtual IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) => - InnerSession.GetStreamingResponseAsync(updates, cancellationToken); + CancellationToken cancellationToken = default) => + InnerSession.GetStreamingResponseAsync(cancellationToken); /// public virtual object? GetService(Type serviceType, object? serviceKey = null) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs index 0d1f6a05a8a..ae27baf91b1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs @@ -27,25 +27,24 @@ public interface IRealtimeSession : IDisposable, IAsyncDisposable RealtimeSessionOptions? Options { get; } /// - /// Injects a client message into the session. + /// Sends a client message to the session. /// - /// The client message to inject. + /// The client message to send. /// A token to cancel the operation. - /// A task that represents the asynchronous injection operation. + /// A task that represents the asynchronous send operation. /// - /// This method allows for the injection of client messages into the session at any time, which can be used to influence the session's behavior or state. + /// This method allows for sending client messages to the session at any time, which can be used to influence the session's behavior or state. /// - Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); + Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); - /// Sends real-time messages and streams the response. - /// The sequence of real-time messages to send. + /// Streams the response from the real-time session. /// A token to cancel the operation. /// The response messages generated by the session. /// /// This method cannot be called multiple times concurrently on the same session instance. /// IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default); + CancellationToken cancellationToken = default); /// Asks the for an object of the specified type . /// The type of object being requested. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index ffb1cd88a30..e3662821608 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -271,7 +271,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken } /// - public async Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -572,40 +572,12 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel /// public async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - - var processUpdatesTask = Task.Run(async () => - { - try - { - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - await InjectClientMessageAsync(message, cancellationToken).ConfigureAwait(false); - } - } - catch (OperationCanceledException) - { - // Expected when the session is cancelled. - } - catch (ObjectDisposedException) - { - // Expected when the session is disposed concurrently. - } - catch (WebSocketException) - { - // Expected when the WebSocket is in an aborted state. - } - }, cancellationToken); - await foreach (var serverEvent in _eventChannel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return serverEvent; } - - await processUpdatesTask.ConfigureAwait(false); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs index a30c103ad4f..6566f0deb44 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSession { /// The delegate to use as the implementation of . - private readonly Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> _getStreamingResponseFunc; + private readonly Func> _getStreamingResponseFunc; /// /// Initializes a new instance of the class. @@ -27,7 +27,7 @@ internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSes /// is . public AnonymousDelegatingRealtimeSession( IRealtimeSession innerSession, - Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> getStreamingResponseFunc) + Func> getStreamingResponseFunc) : base(innerSession) { _getStreamingResponseFunc = Throw.IfNull(getStreamingResponseFunc); @@ -35,10 +35,8 @@ public AnonymousDelegatingRealtimeSession( /// public override IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - - return _getStreamingResponseFunc(updates, InnerSession, cancellationToken); + return _getStreamingResponseFunc(InnerSession, cancellationToken); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 452a71bec98..87ca8f7efdf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -250,10 +250,8 @@ public int MaximumConsecutiveErrorsPerRequest /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - // Create an activity to group function invocations together for better observability. using Activity? activity = FunctionInvocationHelpers.CurrentActivityIsInvokeAgent ? null : _activitySource?.StartActivity(OpenTelemetryConsts.GenAI.OrchestrateToolsName); @@ -262,7 +260,7 @@ public override async IAsyncEnumerable GetStreamingRespon int consecutiveErrorCount = 0; int iterationCount = 0; - await foreach (var message in InnerSession.GetStreamingResponseAsync(updates, cancellationToken).ConfigureAwait(false)) + await foreach (var message in InnerSession.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) { // Check if this message contains function calls bool hasFunctionCalls = false; @@ -307,7 +305,7 @@ public override async IAsyncEnumerable GetStreamingRespon foreach (var resultMessage in results.functionResults) { // inject back the function result messages to the inner session - await InnerSession.InjectClientMessageAsync(resultMessage, cancellationToken).ConfigureAwait(false); + await InnerSession.SendClientMessageAsync(resultMessage, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index 520b0748b09..34ece9a8df9 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -90,7 +90,7 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } /// - public override async Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -98,41 +98,39 @@ public override async Task InjectClientMessageAsync(RealtimeClientMessage messag { if (_logger.IsEnabled(LogLevel.Trace)) { - LogInjectMessageSensitive(GetLoggableString(message)); + LogSendMessageSensitive(GetLoggableString(message)); } else { - LogInjectMessage(); + LogSendMessage(); } } try { - await base.InjectClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); if (_logger.IsEnabled(LogLevel.Debug)) { - LogCompleted(nameof(InjectClientMessageAsync)); + LogCompleted(nameof(SendClientMessageAsync)); } } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(InjectClientMessageAsync)); + LogInvocationCanceled(nameof(SendClientMessageAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(InjectClientMessageAsync), ex); + LogInvocationFailed(nameof(SendClientMessageAsync), ex); throw; } } /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - if (_logger.IsEnabled(LogLevel.Debug)) { LogInvoked(nameof(GetStreamingResponseAsync)); @@ -141,7 +139,7 @@ public override async IAsyncEnumerable GetStreamingRespon IAsyncEnumerator e; try { - e = base.GetStreamingResponseAsync(WrapUpdatesWithLoggingAsync(updates, cancellationToken), cancellationToken).GetAsyncEnumerator(cancellationToken); + e = base.GetStreamingResponseAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); } catch (OperationCanceledException) { @@ -248,28 +246,6 @@ private string GetLoggableString(RealtimeServerMessage message) return obj.ToJsonString(); } - private async IAsyncEnumerable WrapUpdatesWithLoggingAsync( - IAsyncEnumerable updates, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogStreamingClientMessageSensitive(GetLoggableString(message)); - } - else - { - LogStreamingClientMessage(); - } - } - - yield return message; - } - } - private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] @@ -278,21 +254,15 @@ private async IAsyncEnumerable WrapUpdatesWithLoggingAsyn [LoggerMessage(LogLevel.Trace, "{MethodName} invoked: Options: {Options}.")] private partial void LogInvokedSensitive(string methodName, string options); - [LoggerMessage(LogLevel.Debug, "InjectClientMessageAsync invoked.")] - private partial void LogInjectMessage(); + [LoggerMessage(LogLevel.Debug, "SendClientMessageAsync invoked.")] + private partial void LogSendMessage(); - [LoggerMessage(LogLevel.Trace, "InjectClientMessageAsync invoked: Message: {Message}.")] - private partial void LogInjectMessageSensitive(string message); + [LoggerMessage(LogLevel.Trace, "SendClientMessageAsync invoked: Message: {Message}.")] + private partial void LogSendMessageSensitive(string message); [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] private partial void LogCompleted(string methodName); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync sending client message.")] - private partial void LogStreamingClientMessage(); - - [LoggerMessage(LogLevel.Trace, "GetStreamingResponseAsync sending client message: {ClientMessage}")] - private partial void LogStreamingClientMessageSensitive(string clientMessage); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync received server message.")] private partial void LogStreamingServerMessage(); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index bcb76facb5a..316a107d9f2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -199,11 +199,42 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } } + /// + public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + { + if (EnableSensitiveData && _activitySource.HasListeners()) + { + var otelMessage = ExtractClientOtelMessage(message); + + if (otelMessage is not null) + { + RealtimeSessionOptions? options = Options; + string? requestModelId = options?.Model ?? _defaultModelId; + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + using Activity? inputActivity = CreateAndConfigureActivity(options: null); + if (inputActivity is { IsAllDataRequested: true }) + { + _ = inputActivity.AddTag(OpenTelemetryConsts.GenAI.Input.Messages, SerializeMessage(otelMessage)); + } + + // Record metrics + if (_operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = default; + AddMetricTags(ref tags, requestModelId, responseModelId: null); + _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + } + } + + await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + } + /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); _jsonSerializerOptions.MakeReadOnly(); RealtimeSessionOptions? options = Options; @@ -215,15 +246,10 @@ public override async IAsyncEnumerable GetStreamingRespon // Determine if we should capture messages for telemetry bool captureMessages = EnableSensitiveData && _activitySource.HasListeners(); - // Wrap client messages to capture input content and create input activity - IAsyncEnumerable wrappedUpdates = captureMessages - ? WrapClientMessagesForTelemetryAsync(updates, options, cancellationToken) - : updates; - IAsyncEnumerable responses; try { - responses = base.GetStreamingResponseAsync(wrappedUpdates, cancellationToken); + responses = base.GetStreamingResponseAsync(cancellationToken); } catch (Exception ex) { @@ -307,42 +333,6 @@ public override async IAsyncEnumerable GetStreamingRespon } } - /// Wraps client messages to capture content for telemetry with its own activity. - private async IAsyncEnumerable WrapClientMessagesForTelemetryAsync( - IAsyncEnumerable updates, - RealtimeSessionOptions? options, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - string? requestModelId = options?.Model ?? _defaultModelId; - - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Capture input content from current client message - var otelMessage = ExtractClientOtelMessage(message); - - // Only create activity when there's content to log - if (otelMessage is not null) - { - using Activity? inputActivity = CreateAndConfigureActivity(options: null); - if (inputActivity is { IsAllDataRequested: true }) - { - _ = inputActivity.AddTag(OpenTelemetryConsts.GenAI.Input.Messages, SerializeMessage(otelMessage)); - } - - // Record metrics - if (_operationDurationHistogram.Enabled && stopwatch is not null) - { - TagList tags = default; - AddMetricTags(ref tags, requestModelId, responseModelId: null); - _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); - } - } - - yield return message; - } - } - /// Adds output modalities tag to the activity. private static void AddOutputModalitiesTag(Activity? activity, HashSet? outputModalities) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs index 545c68b9d77..de02b3dbe1d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs @@ -92,8 +92,8 @@ public RealtimeSessionBuilder Use(Func /// /// A delegate that provides the implementation for . - /// This delegate is invoked with the sequence of realtime client messages, a delegate that represents invoking - /// the inner session, and a cancellation token. The delegate should be passed whatever client messages and + /// This delegate is invoked with a delegate that represents invoking + /// the inner session, and a cancellation token. The delegate should be passed whatever /// cancellation token should be passed along to the next stage in the pipeline. /// /// The updated instance. @@ -103,7 +103,7 @@ public RealtimeSessionBuilder Use(Func /// is . public RealtimeSessionBuilder Use( - Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> getStreamingResponseFunc) + Func> getStreamingResponseFunc) { _ = Throw.IfNull(getStreamingResponseFunc); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs index 4dc48292694..4ec5d6e03a8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs @@ -14,11 +14,11 @@ public sealed class TestRealtimeSession : IRealtimeSession /// Gets or sets the callback to invoke when is called. public Func? UpdateAsyncCallback { get; set; } - /// Gets or sets the callback to invoke when is called. - public Func? InjectClientMessageAsyncCallback { get; set; } + /// Gets or sets the callback to invoke when is called. + public Func? SendClientMessageAsyncCallback { get; set; } /// Gets or sets the callback to invoke when is called. - public Func, CancellationToken, IAsyncEnumerable>? GetStreamingResponseAsyncCallback { get; set; } + public Func>? GetStreamingResponseAsyncCallback { get; set; } /// Gets or sets the callback to invoke when is called. public Func? GetServiceCallback { get; set; } @@ -33,16 +33,16 @@ public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancel } /// - public Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { - return InjectClientMessageAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; + return SendClientMessageAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; } /// public IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - return GetStreamingResponseAsyncCallback?.Invoke(updates, cancellationToken) ?? EmptyAsyncEnumerable(); + return GetStreamingResponseAsyncCallback?.Invoke(cancellationToken) ?? EmptyAsyncEnumerable(); } /// diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index d03c0621dd3..6ec0ed0fa8c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -56,38 +56,24 @@ public async Task UpdateAsync_NullOptions_Throws() } [Fact] - public async Task InjectClientMessageAsync_NullMessage_Throws() + public async Task SendClientMessageAsync_NullMessage_Throws() { using var session = new OpenAIRealtimeSession("key", "model"); - await Assert.ThrowsAsync("message", () => session.InjectClientMessageAsync(null!)); + await Assert.ThrowsAsync("message", () => session.SendClientMessageAsync(null!)); } [Fact] - public async Task InjectClientMessageAsync_CancelledToken_ReturnsSilently() + public async Task SendClientMessageAsync_CancelledToken_ReturnsSilently() { using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); // Should not throw when cancellation is requested. - await session.InjectClientMessageAsync(new RealtimeClientMessage(), cts.Token); + await session.SendClientMessageAsync(new RealtimeClientMessage(), cts.Token); Assert.Null(session.Options); } - [Fact] - public async Task GetStreamingResponseAsync_NullUpdates_Throws() - { - using var session = new OpenAIRealtimeSession("key", "model"); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var msg in session.GetStreamingResponseAsync(null!)) - { - _ = msg; - } - }); - } - [Fact] public async Task ConnectAsync_CancelledToken_ReturnsFalse() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index a9f6df9b410..9241a02fbbd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -51,13 +51,13 @@ public async Task UpdateAsync_DelegatesToInner() } [Fact] - public async Task InjectClientMessageAsync_DelegatesToInner() + public async Task SendClientMessageAsync_DelegatesToInner() { var called = false; var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; using var inner = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { Assert.Same(sentMessage, msg); called = true; @@ -66,7 +66,7 @@ public async Task InjectClientMessageAsync_DelegatesToInner() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.InjectClientMessageAsync(sentMessage); + await delegating.SendClientMessageAsync(sentMessage); Assert.True(called); } @@ -76,12 +76,12 @@ public async Task GetStreamingResponseAsync_DelegatesToInner() var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(expected, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldSingle(expected, ct), }; using var delegating = new NoOpDelegatingRealtimeSession(inner); var messages = new List(); - await foreach (var msg in delegating.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in delegating.GetStreamingResponseAsync()) { messages.Add(msg); } @@ -164,7 +164,7 @@ public async Task UpdateAsync_FlowsCancellationToken() } [Fact] - public async Task InjectClientMessageAsync_FlowsCancellationToken() + public async Task SendClientMessageAsync_FlowsCancellationToken() { CancellationToken capturedToken = default; using var cts = new CancellationTokenSource(); @@ -172,7 +172,7 @@ public async Task InjectClientMessageAsync_FlowsCancellationToken() using var inner = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (msg, ct) => + SendClientMessageAsyncCallback = (msg, ct) => { capturedToken = ct; return Task.CompletedTask; @@ -180,18 +180,10 @@ public async Task InjectClientMessageAsync_FlowsCancellationToken() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.InjectClientMessageAsync(sentMessage, cts.Token); + await delegating.SendClientMessageAsync(sentMessage, cts.Token); Assert.Equal(cts.Token, capturedToken); } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldSingle( RealtimeServerMessage message, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -224,10 +216,10 @@ public DisposableTestRealtimeSession(Action onDispose) public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; public IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) => + CancellationToken cancellationToken = default) => EmptyUpdatesServer(cancellationToken); public object? GetService(Type serviceType, object? serviceKey = null) => null; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index fb953217bc3..173cda9ee60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -61,21 +61,6 @@ public void MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() Assert.Equal(0, session.MaximumConsecutiveErrorsPerRequest); } - [Fact] - public async Task GetStreamingResponseAsync_NullUpdates_Throws() - { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var msg in session.GetStreamingResponseAsync(null!)) - { - _ = msg; - } - }); - } - [Fact] public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() { @@ -87,12 +72,12 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(serverMessages, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(serverMessages, ct), }; using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -114,11 +99,11 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [getWeather] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_001", "get_weather", new Dictionary { ["city"] = "Seattle" }), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -128,7 +113,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -162,11 +147,11 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_002", "get_weather", new Dictionary { ["city"] = "London" }), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -178,7 +163,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() AdditionalTools = [getWeather], }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -208,8 +193,8 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [countFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(messages, ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -218,7 +203,7 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -242,11 +227,11 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [myFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_custom", "my_func", null), ], ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -258,7 +243,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() }, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -272,11 +257,11 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -285,7 +270,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( using var session = new FunctionInvokingRealtimeSession(inner); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -309,11 +294,11 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_fail", "fail_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -325,7 +310,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors IncludeDetailedErrors = true, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -348,11 +333,11 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_fail2", "fail_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -364,7 +349,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna IncludeDetailedErrors = false, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -392,12 +377,12 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -410,7 +395,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -428,11 +413,11 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -445,7 +430,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -513,8 +498,8 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [slowFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages([combinedMessage], ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages([combinedMessage], ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -522,7 +507,7 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall AllowConcurrentInvocation = true, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -556,8 +541,8 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(messages, ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -568,7 +553,7 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw // Should eventually throw after exceeding the consecutive error limit await Assert.ThrowsAsync(async () => { - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -618,12 +603,12 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [declaration] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_decl", "my_declaration", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -633,7 +618,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -662,14 +647,6 @@ private static RealtimeServerResponseOutputItemMessage CreateFunctionCallOutputI }; } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldMessages( IList messages, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 5a26dc339d8..9eac51a59ea 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -90,7 +90,7 @@ public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) [InlineData(LogLevel.Trace)] [InlineData(LogLevel.Debug)] [InlineData(LogLevel.Information)] - public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel level) + public async Task SendClientMessageAsync_LogsInvocationAndCompletion(LogLevel level) { var collector = new FakeLogCollector(); @@ -100,7 +100,7 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => Task.CompletedTask, + SendClientMessageAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; using var session = innerSession @@ -108,20 +108,20 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel .UseLogging() .Build(services); - await session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); + await session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) { Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked:", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked:", entry.Message), + entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); } else if (level is LogLevel.Debug) { Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); } else { @@ -140,7 +140,7 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => GetMessagesAsync() + GetStreamingResponseAsyncCallback = (cancellationToken) => GetMessagesAsync() }; static async IAsyncEnumerable GetMessagesAsync() @@ -155,7 +155,7 @@ static async IAsyncEnumerable GetMessagesAsync() .UseLogging(loggerFactory) .Build(); - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync())) + await foreach (var message in session.GetStreamingResponseAsync()) { // nop } @@ -250,7 +250,7 @@ public async Task GetStreamingResponseAsync_LogsCancellation() using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => ThrowCancellationAsync(cancellationToken) + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowCancellationAsync(cancellationToken) }; static async IAsyncEnumerable ThrowCancellationAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -270,7 +270,7 @@ static async IAsyncEnumerable ThrowCancellationAsync([Enu cts.Cancel(); await Assert.ThrowsAsync(async () => { - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync(), cts.Token)) + await foreach (var message in session.GetStreamingResponseAsync(cts.Token)) { // nop } @@ -290,7 +290,7 @@ public async Task GetStreamingResponseAsync_LogsErrors() using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => ThrowErrorAsync() + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowErrorAsync() }; static async IAsyncEnumerable ThrowErrorAsync() @@ -309,7 +309,7 @@ static async IAsyncEnumerable ThrowErrorAsync() await Assert.ThrowsAsync(async () => { - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync())) + await foreach (var message in session.GetStreamingResponseAsync()) { // nop } @@ -338,7 +338,7 @@ public void GetService_ReturnsLoggingSessionWhenRequested() } [Fact] - public async Task InjectClientMessageAsync_LogsCancellation() + public async Task SendClientMessageAsync_LogsCancellation() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); @@ -347,7 +347,7 @@ public async Task InjectClientMessageAsync_LogsCancellation() using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => + SendClientMessageAsyncCallback = (message, cancellationToken) => { throw new OperationCanceledException(cancellationToken); }, @@ -360,23 +360,23 @@ public async Task InjectClientMessageAsync_LogsCancellation() cts.Cancel(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); + session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync canceled.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.Contains("SendClientMessageAsync canceled.", entry.Message)); } [Fact] - public async Task InjectClientMessageAsync_LogsErrors() + public async Task SendClientMessageAsync_LogsErrors() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => + SendClientMessageAsyncCallback = (message, cancellationToken) => { throw new InvalidOperationException("Inject error"); }, @@ -388,70 +388,12 @@ public async Task InjectClientMessageAsync_LogsErrors() .Build(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage())); + session.SendClientMessageAsync(new RealtimeClientMessage())); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.True(entry.Message.Contains("InjectClientMessageAsync failed.") && entry.Level == LogLevel.Error)); - } - - [Theory] - [InlineData(LogLevel.Trace)] - [InlineData(LogLevel.Debug)] - [InlineData(LogLevel.Information)] - public async Task GetStreamingResponseAsync_LogsClientMessages(LogLevel level) - { - var collector = new FakeLogCollector(); - using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); - - using var innerSession = new TestRealtimeSession - { - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ConsumeAndYield(updates, cancellationToken) - }; - - static async IAsyncEnumerable ConsumeAndYield( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) - { - await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // consume - } - - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone }; - } - - using var session = innerSession - .AsBuilder() - .UseLogging(loggerFactory) - .Build(); - - var clientMessages = GetClientMessages(); - await foreach (var message in session.GetStreamingResponseAsync(clientMessages)) - { - // consume - } - - var logs = collector.GetSnapshot(); - if (level is LogLevel.Trace) - { - // Should log: invoked, client message (sensitive), server message (sensitive), completed - Assert.True(logs.Count >= 3); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync invoked.")); - Assert.Contains(logs, entry => entry.Message.Contains("sending client message:")); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync completed.")); - } - else if (level is LogLevel.Debug) - { - Assert.True(logs.Count >= 3); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync invoked.")); - Assert.Contains(logs, entry => entry.Message.Contains("sending client message.")); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync completed.")); - } - else - { - Assert.Empty(logs); - } + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.True(entry.Message.Contains("SendClientMessageAsync failed.") && entry.Level == LogLevel.Error)); } [Fact] @@ -500,17 +442,4 @@ public void UseLogging_ConfigureCallback_IsInvoked() Assert.True(configured); } - private static async IAsyncEnumerable EmptyAsyncEnumerableAsync() - { - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - - private static async IAsyncEnumerable GetClientMessages( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield return new RealtimeClientMessage { MessageId = "client_evt_1" }; - } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 79552329be4..5494b202a3a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -44,19 +44,13 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl GetServiceCallback = (serviceType, serviceKey) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("testprovider", new Uri("http://localhost:12345/realtime"), "gpt-4-realtime") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackAsync(cancellationToken), }; - static async IAsyncEnumerable CallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable CallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - // Consume the incoming updates - await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Just consume the update - } + _ = cancellationToken; yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; @@ -91,8 +85,12 @@ static async IAsyncEnumerable CallbackAsync( }) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -275,13 +273,13 @@ public async Task GetStreamingResponseAsync_TracesError() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ThrowingCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowingCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable ThrowingCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable ThrowingCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); + _ = cancellationToken; yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated }; throw new InvalidOperationException("Streaming error"); } @@ -291,10 +289,9 @@ static async IAsyncEnumerable ThrowingCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); await Assert.ThrowsAsync(async () => { - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -319,18 +316,13 @@ public async Task GetStreamingResponseAsync_TracesErrorFromResponse() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ErrorResponseCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => ErrorResponseCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable ErrorResponseCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable ErrorResponseCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) { @@ -345,8 +337,12 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -374,18 +370,13 @@ public async Task DefaultVoiceSpeed_NotLogged() Model = "test-model", VoiceSpeed = 1.0, // Default value should not be logged }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => EmptyCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable EmptyCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable EmptyCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } @@ -395,8 +386,12 @@ static async IAsyncEnumerable EmptyCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -417,18 +412,15 @@ public async Task NoListeners_NoActivityCreated() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => EmptyCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable EmptyCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning disable S4144 // Methods should not have identical implementations + static async IAsyncEnumerable EmptyCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning restore S4144 { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield break; } @@ -440,9 +432,8 @@ static async IAsyncEnumerable EmptyCallbackAsync( .Build(); // This should work without errors even without listeners - var clientMessages = GetClientMessagesAsync(); var count = 0; - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var response in session.GetStreamingResponseAsync()) { count++; } @@ -469,21 +460,6 @@ public async Task UpdateAsync_InvalidArgs_Throws() await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); } - [Fact] - public async Task GetStreamingResponseAsync_InvalidArgs_Throws() - { - using var innerSession = new TestRealtimeSession(); - using var session = new OpenTelemetryRealtimeSession(innerSession); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var _ in session.GetStreamingResponseAsync(null!)) - { - // Should not reach here - } - }); - } - [Fact] public void GetService_ReturnsActivitySource() { @@ -525,18 +501,13 @@ public async Task TranscriptionSessionKind_Logged() Model = "whisper-1", SessionKind = RealtimeSessionKind.Transcription, }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => TranscriptionCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => TranscriptionCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable TranscriptionCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable TranscriptionCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { @@ -550,8 +521,12 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -589,7 +564,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) ToolMode = mode, Tools = [AIFunctionFactory.Create((string query) => query, "Search")], }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -597,8 +572,12 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -624,7 +603,7 @@ public async Task AIFunction_ForcedTool_Logged() Model = "test-model", ToolMode = ChatToolMode.RequireSpecific("SpecificSearch"), }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -632,8 +611,12 @@ public async Task AIFunction_ForcedTool_Logged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -660,7 +643,7 @@ public async Task RequireAny_ToolMode_Logged() Model = "test-model", ToolMode = ChatToolMode.RequireAny, }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -668,8 +651,12 @@ public async Task RequireAny_ToolMode_Logged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -695,7 +682,7 @@ public async Task NoToolChoice_NotLogged() { Model = "test-model", }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -703,8 +690,12 @@ public async Task NoToolChoice_NotLogged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -728,7 +719,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -736,7 +727,12 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithToolResultAsync())) + await foreach (var msg in GetClientMessagesWithToolResultAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -766,7 +762,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithToolCallAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; using var session = innerSession @@ -774,7 +770,12 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -804,7 +805,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithToolCallAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; using var session = innerSession @@ -812,7 +813,12 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = false) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithToolResultAsync())) + await foreach (var msg in GetClientMessagesWithToolResultAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -822,15 +828,12 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() Assert.Null(activity.GetTagItem("gen_ai.output.messages")); } - private static async IAsyncEnumerable SimpleCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning disable S4144 // Methods should not have identical implementations + private static async IAsyncEnumerable SimpleCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning restore S4144 { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } @@ -855,16 +858,10 @@ private static async IAsyncEnumerable GetClientMessagesWi yield return new RealtimeClientResponseCreateMessage(); } - private static async IAsyncEnumerable CallbackWithToolCallAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithToolCallAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - // Consume incoming messages - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; // Yield a function call item from the server using RealtimeServerResponseOutputItemMessage var contentItem = new RealtimeContentItem( @@ -893,7 +890,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -901,7 +898,12 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -931,7 +933,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -939,7 +941,12 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -969,7 +976,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -977,7 +984,12 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithInstructionsAsync())) + await foreach (var msg in GetClientMessagesWithInstructionsAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1006,7 +1018,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1014,7 +1026,12 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithItemsAsync())) + await foreach (var msg in GetClientMessagesWithItemsAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1043,7 +1060,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithTextOutputAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTextOutputAsync(cancellationToken), }; using var session = innerSession @@ -1051,7 +1068,12 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1079,7 +1101,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithTranscriptionAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTranscriptionAsync(cancellationToken), }; using var session = innerSession @@ -1087,7 +1109,12 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1115,7 +1142,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithServerErrorAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithServerErrorAsync(cancellationToken), }; using var session = innerSession @@ -1123,7 +1150,12 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1151,7 +1183,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1159,7 +1191,12 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithTextContentAsync())) + await foreach (var msg in GetClientMessagesWithTextContentAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1187,7 +1224,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1195,7 +1232,12 @@ public async Task DataContentInClientMessage_LoggedWithModality() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithImageContentAsync())) + await foreach (var msg in GetClientMessagesWithImageContentAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1247,15 +1289,10 @@ private static async IAsyncEnumerable GetClientMessagesWi yield return new RealtimeClientResponseCreateMessage(); } - private static async IAsyncEnumerable CallbackWithTextOutputAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithTextOutputAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) { @@ -1264,15 +1301,10 @@ private static async IAsyncEnumerable CallbackWithTextOut yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - private static async IAsyncEnumerable CallbackWithTranscriptionAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithTranscriptionAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { @@ -1281,15 +1313,10 @@ private static async IAsyncEnumerable CallbackWithTranscr yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - private static async IAsyncEnumerable CallbackWithServerErrorAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithServerErrorAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerErrorMessage { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index 25b10374d75..b39e698c57f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -64,7 +64,7 @@ public void Use_StreamingDelegate_NullFunc_Throws() Assert.Throws( "getStreamingResponseFunc", - () => builder.Use((Func, IRealtimeSession, CancellationToken, IAsyncEnumerable>)null!)); + () => builder.Use((Func>)null!)); } [Fact] @@ -142,18 +142,18 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() var intercepted = false; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; var builder = new RealtimeSessionBuilder(inner); - builder.Use((updates, innerSession, ct) => + builder.Use((innerSession, ct) => { intercepted = true; - return innerSession.GetStreamingResponseAsync(updates, ct); + return innerSession.GetStreamingResponseAsync(ct); }); using var pipeline = builder.Build(); - await foreach (var msg in pipeline.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in pipeline.GetStreamingResponseAsync()) { Assert.Equal("inner", msg.MessageId); } @@ -177,14 +177,6 @@ public void AsBuilder_ReturnsBuilder() Assert.Same(inner, builder.Build()); } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldSingle( RealtimeServerMessage message, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -209,11 +201,10 @@ public OrderTrackingSession(IRealtimeSession inner, string name, List ca public IRealtimeSession GetInner() => InnerSession; public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) { _callOrder.Add(Name); - await foreach (var msg in base.GetStreamingResponseAsync(updates, cancellationToken).ConfigureAwait(false)) + await foreach (var msg in base.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) { yield return msg; } From 02fd006ca38b9f3817f80f0e6d51ea565287c46b Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 12:23:12 -0800 Subject: [PATCH 37/38] Make RealtimeSessionOptions and related types immutable with init-only properties and IReadOnlyList --- .../Realtime/RealtimeAudioFormat.cs | 8 +- .../Realtime/RealtimeSessionOptions.cs | 60 +++--- .../SemanticVoiceActivityDetection.cs | 4 +- .../Realtime/ServerVoiceActivityDetection.cs | 16 +- .../Realtime/VoiceActivityDetection.cs | 8 +- .../OpenAIRealtimeSession.cs | 185 ++++++++---------- .../FunctionInvokingRealtimeSession.cs | 4 +- .../Realtime/RealtimeSessionOptionsTests.cs | 40 ++-- 8 files changed, 153 insertions(+), 172 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs index 93c97d08fd2..3d8962c6780 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -22,16 +22,16 @@ public RealtimeAudioFormat(string mediaType, int sampleRate) } /// - /// Gets or sets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). + /// Gets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). /// - public string MediaType { get; set; } + public string MediaType { get; init; } /// - /// Gets or sets the sample rate of the audio in Hertz. + /// Gets the sample rate of the audio in Hertz. /// /// /// When constructed via , this property is always set. /// The nullable type allows deserialized instances to omit the sample rate when the server does not provide one. /// - public int? SampleRate { get; set; } + public int? SampleRate { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 6180a12ab87..03a02b73187 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -14,84 +14,84 @@ namespace Microsoft.Extensions.AI; public class RealtimeSessionOptions { /// - /// Gets or sets the session kind. + /// Gets the session kind. /// /// /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. /// - public RealtimeSessionKind SessionKind { get; set; } = RealtimeSessionKind.Realtime; + public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Realtime; /// - /// Gets or sets the model name to use for the session. + /// Gets the model name to use for the session. /// - public string? Model { get; set; } + public string? Model { get; init; } /// - /// Gets or sets the input audio format for the session. + /// Gets the input audio format for the session. /// - public RealtimeAudioFormat? InputAudioFormat { get; set; } + public RealtimeAudioFormat? InputAudioFormat { get; init; } /// - /// Gets or sets the noise reduction options for the session. + /// Gets the noise reduction options for the session. /// - public NoiseReductionOptions? NoiseReductionOptions { get; set; } + public NoiseReductionOptions? NoiseReductionOptions { get; init; } /// - /// Gets or sets the transcription options for the session. + /// Gets the transcription options for the session. /// - public TranscriptionOptions? TranscriptionOptions { get; set; } + public TranscriptionOptions? TranscriptionOptions { get; init; } /// - /// Gets or sets the voice activity detection options for the session. + /// Gets the voice activity detection options for the session. /// - public VoiceActivityDetection? VoiceActivityDetection { get; set; } + public VoiceActivityDetection? VoiceActivityDetection { get; init; } /// - /// Gets or sets the output audio format for the session. + /// Gets the output audio format for the session. /// - public RealtimeAudioFormat? OutputAudioFormat { get; set; } + public RealtimeAudioFormat? OutputAudioFormat { get; init; } /// - /// Gets or sets the output voice speed for the session. + /// Gets the output voice speed for the session. /// /// /// The default value is 1.0, which represents normal speed. /// - public double VoiceSpeed { get; set; } = 1.0; + public double VoiceSpeed { get; init; } = 1.0; /// - /// Gets or sets the output voice for the session. + /// Gets the output voice for the session. /// - public string? Voice { get; set; } + public string? Voice { get; init; } /// - /// Gets or sets the default system instructions for the session. + /// Gets the default system instructions for the session. /// - public string? Instructions { get; set; } + public string? Instructions { get; init; } /// - /// Gets or sets the maximum number of response tokens for the session. + /// Gets the maximum number of response tokens for the session. /// - public int? MaxOutputTokens { get; set; } + public int? MaxOutputTokens { get; init; } /// - /// Gets or sets the output modalities for the response. like "text", "audio". + /// Gets the output modalities for the response. like "text", "audio". /// If null, then default conversation modalities will be used. /// - public IList? OutputModalities { get; set; } + public IReadOnlyList? OutputModalities { get; init; } /// - /// Gets or sets the tool choice mode for the session. + /// Gets the tool choice mode for the session. /// - public ChatToolMode? ToolMode { get; set; } + public ChatToolMode? ToolMode { get; init; } /// - /// Gets or sets the AI tools available for generating the response. + /// Gets the AI tools available for generating the response. /// - public IList? Tools { get; set; } + public IReadOnlyList? Tools { get; init; } /// - /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. + /// Gets a callback responsible for creating the raw representation of the session options from an underlying implementation. /// /// /// The underlying implementation might have its own representation of options. @@ -108,5 +108,5 @@ public class RealtimeSessionOptions /// properties on . /// [JsonIgnore] - public Func? RawRepresentationFactory { get; set; } + public Func? RawRepresentationFactory { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs index 12f995d5b42..c4c94f3b7f5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI; public class SemanticVoiceActivityDetection : VoiceActivityDetection { /// - /// Gets or sets the eagerness level for semantic voice activity detection. + /// Gets the eagerness level for semantic voice activity detection. /// - public SemanticEagerness Eagerness { get; set; } = SemanticEagerness.Auto; + public SemanticEagerness Eagerness { get; init; } = SemanticEagerness.Auto; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs index bdea1f39cbb..7b0946337ef 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs @@ -13,25 +13,25 @@ namespace Microsoft.Extensions.AI; public class ServerVoiceActivityDetection : VoiceActivityDetection { /// - /// Gets or sets the idle timeout in milliseconds to detect the end of speech. + /// Gets the idle timeout in milliseconds to detect the end of speech. /// - public int IdleTimeoutInMilliseconds { get; set; } + public int IdleTimeoutInMilliseconds { get; init; } /// - /// Gets or sets the prefix padding in milliseconds to include before detected speech. + /// Gets the prefix padding in milliseconds to include before detected speech. /// - public int PrefixPaddingInMilliseconds { get; set; } = 300; + public int PrefixPaddingInMilliseconds { get; init; } = 300; /// - /// Gets or sets the silence duration in milliseconds to consider as a pause. + /// Gets the silence duration in milliseconds to consider as a pause. /// - public int SilenceDurationInMilliseconds { get; set; } = 500; + public int SilenceDurationInMilliseconds { get; init; } = 500; /// - /// Gets or sets the threshold for voice activity detection. + /// Gets the threshold for voice activity detection. /// /// /// A value between 0.0 and 1.0, where higher values make the detection more sensitive. /// - public double Threshold { get; set; } = 0.5; + public double Threshold { get; init; } = 0.5; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs index d244cf24307..aa6c58c00f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs @@ -13,12 +13,12 @@ namespace Microsoft.Extensions.AI; public class VoiceActivityDetection { /// - /// Gets or sets a value indicating whether to create a response when voice activity is detected. + /// Gets a value indicating whether to create a response when voice activity is detected. /// - public bool CreateResponse { get; set; } + public bool CreateResponse { get; init; } /// - /// Gets or sets a value indicating whether to interrupt the response when voice activity is detected. + /// Gets a value indicating whether to interrupt the response when voice activity is detected. /// - public bool InterruptResponse { get; set; } + public bool InterruptResponse { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index e3662821608..52571623ead 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1315,17 +1315,7 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati { if (root.TryGetProperty("session", out var sessionElement)) { - var newOptions = DeserializeSessionOptions(sessionElement); - - // Preserve client-side properties that the server cannot round-trip - // as typed objects (tools are returned as JSON schemas, not AITool instances). - if (Options is not null) - { - newOptions.Tools = Options.Tools; - newOptions.ToolMode = Options.ToolMode; - } - - Options = newOptions; + Options = DeserializeSessionOptions(sessionElement, Options); } return new RealtimeServerMessage @@ -1335,37 +1325,35 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati }; } - private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement session) + private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement session, RealtimeSessionOptions? previousOptions) { - var options = new RealtimeSessionOptions(); - - if (session.TryGetProperty("type", out var typeElement)) + RealtimeSessionKind sessionKind = RealtimeSessionKind.Realtime; + if (session.TryGetProperty("type", out var typeElement) && typeElement.GetString() == "transcription") { - options.SessionKind = typeElement.GetString() == "transcription" - ? RealtimeSessionKind.Transcription - : RealtimeSessionKind.Realtime; + sessionKind = RealtimeSessionKind.Transcription; } - if (session.TryGetProperty("model", out var modelElement)) - { - options.Model = modelElement.GetString(); - } + string? model = session.TryGetProperty("model", out var modelElement) ? modelElement.GetString() : null; - if (session.TryGetProperty("instructions", out var instructionsElement) && - instructionsElement.ValueKind == JsonValueKind.String) - { - options.Instructions = instructionsElement.GetString(); - } + string? instructions = session.TryGetProperty("instructions", out var instructionsElement) && instructionsElement.ValueKind == JsonValueKind.String + ? instructionsElement.GetString() + : null; - if (session.TryGetProperty("max_output_tokens", out var maxTokensElement)) - { - options.MaxOutputTokens = ParseMaxOutputTokens(maxTokensElement); - } + int? maxOutputTokens = session.TryGetProperty("max_output_tokens", out var maxTokensElement) + ? ParseMaxOutputTokens(maxTokensElement) + : null; - if (session.TryGetProperty("output_modalities", out var modalitiesElement)) - { - options.OutputModalities = ParseOutputModalities(modalitiesElement); - } + IReadOnlyList? outputModalities = session.TryGetProperty("output_modalities", out var modalitiesElement) + ? ParseOutputModalities(modalitiesElement) + : null; + + RealtimeAudioFormat? inputAudioFormat = null; + NoiseReductionOptions? noiseReductionOptions = null; + TranscriptionOptions? transcriptionOptions = null; + VoiceActivityDetection? voiceActivityDetection = null; + RealtimeAudioFormat? outputAudioFormat = null; + double voiceSpeed = 1.0; + string? voice = null; // Audio configuration. if (session.TryGetProperty("audio", out var audioElement) && @@ -1377,14 +1365,14 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess { if (inputElement.TryGetProperty("format", out var inputFormatElement)) { - options.InputAudioFormat = ParseAudioFormat(inputFormatElement); + inputAudioFormat = ParseAudioFormat(inputFormatElement); } if (inputElement.TryGetProperty("noise_reduction", out var noiseElement) && noiseElement.ValueKind == JsonValueKind.Object && noiseElement.TryGetProperty("type", out var noiseTypeElement)) { - options.NoiseReductionOptions = noiseTypeElement.GetString() switch + noiseReductionOptions = noiseTypeElement.GetString() switch { "near_field" => NoiseReductionOptions.NearField, "far_field" => NoiseReductionOptions.FarField, @@ -1396,12 +1384,12 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess transcriptionElement.ValueKind == JsonValueKind.Object) { string? language = transcriptionElement.TryGetProperty("language", out var langElement) ? langElement.GetString() : null; - string? model = transcriptionElement.TryGetProperty("model", out var modelEl) ? modelEl.GetString() : null; + string? transcriptionModel = transcriptionElement.TryGetProperty("model", out var modelEl) ? modelEl.GetString() : null; string? prompt = transcriptionElement.TryGetProperty("prompt", out var promptElement) ? promptElement.GetString() : null; - if (language is not null && model is not null) + if (language is not null && transcriptionModel is not null) { - options.TranscriptionOptions = new TranscriptionOptions { SpeechLanguage = language, ModelId = model, Prompt = prompt }; + transcriptionOptions = new TranscriptionOptions { SpeechLanguage = language, ModelId = transcriptionModel, Prompt = prompt }; } } @@ -1410,63 +1398,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess turnDetectionElement.ValueKind == JsonValueKind.Object && turnDetectionElement.TryGetProperty("type", out var vadTypeElement)) { - string? vadType = vadTypeElement.GetString(); - if (vadType == "server_vad") - { - var serverVad = new ServerVoiceActivityDetection(); - if (turnDetectionElement.TryGetProperty("create_response", out var crElement)) - { - serverVad.CreateResponse = crElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("interrupt_response", out var irElement)) - { - serverVad.InterruptResponse = irElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("idle_timeout_ms", out var itElement) && itElement.ValueKind == JsonValueKind.Number) - { - serverVad.IdleTimeoutInMilliseconds = itElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("prefix_padding_ms", out var ppElement) && ppElement.ValueKind == JsonValueKind.Number) - { - serverVad.PrefixPaddingInMilliseconds = ppElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("silence_duration_ms", out var sdElement) && sdElement.ValueKind == JsonValueKind.Number) - { - serverVad.SilenceDurationInMilliseconds = sdElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("threshold", out var thElement) && thElement.ValueKind == JsonValueKind.Number) - { - serverVad.Threshold = thElement.GetDouble(); - } - - options.VoiceActivityDetection = serverVad; - } - else if (vadType == "semantic_vad") - { - var semanticVad = new SemanticVoiceActivityDetection(); - if (turnDetectionElement.TryGetProperty("create_response", out var crElement)) - { - semanticVad.CreateResponse = crElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("interrupt_response", out var irElement)) - { - semanticVad.InterruptResponse = irElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && - eagernessElement.GetString() is string eagerness) - { - semanticVad.Eagerness = new SemanticEagerness(eagerness); - } - - options.VoiceActivityDetection = semanticVad; - } + voiceActivityDetection = ParseVoiceActivityDetection(vadTypeElement.GetString(), turnDetectionElement); } } @@ -1476,30 +1408,79 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess { if (outputElement.TryGetProperty("format", out var outputFormatElement)) { - options.OutputAudioFormat = ParseAudioFormat(outputFormatElement); + outputAudioFormat = ParseAudioFormat(outputFormatElement); } if (outputElement.TryGetProperty("speed", out var speedElement) && speedElement.ValueKind == JsonValueKind.Number) { - options.VoiceSpeed = speedElement.GetDouble(); + voiceSpeed = speedElement.GetDouble(); } if (outputElement.TryGetProperty("voice", out var voiceElement)) { if (voiceElement.ValueKind == JsonValueKind.String) { - options.Voice = voiceElement.GetString(); + voice = voiceElement.GetString(); } else if (voiceElement.ValueKind == JsonValueKind.Object && voiceElement.TryGetProperty("id", out var voiceIdElement)) { - options.Voice = voiceIdElement.GetString(); + voice = voiceIdElement.GetString(); } } } } - return options; + return new RealtimeSessionOptions + { + SessionKind = sessionKind, + Model = model, + Instructions = instructions, + MaxOutputTokens = maxOutputTokens, + OutputModalities = outputModalities, + InputAudioFormat = inputAudioFormat, + NoiseReductionOptions = noiseReductionOptions, + TranscriptionOptions = transcriptionOptions, + VoiceActivityDetection = voiceActivityDetection, + OutputAudioFormat = outputAudioFormat, + VoiceSpeed = voiceSpeed, + Voice = voice, + + // Preserve client-side properties that the server cannot round-trip + // as typed objects (tools are returned as JSON schemas, not AITool instances). + Tools = previousOptions?.Tools, + ToolMode = previousOptions?.ToolMode, + }; + } + + private static VoiceActivityDetection? ParseVoiceActivityDetection(string? vadType, JsonElement turnDetectionElement) + { + if (vadType == "server_vad") + { + return new ServerVoiceActivityDetection + { + CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), + InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), + IdleTimeoutInMilliseconds = turnDetectionElement.TryGetProperty("idle_timeout_ms", out var itElement) && itElement.ValueKind == JsonValueKind.Number ? itElement.GetInt32() : 0, + PrefixPaddingInMilliseconds = turnDetectionElement.TryGetProperty("prefix_padding_ms", out var ppElement) && ppElement.ValueKind == JsonValueKind.Number ? ppElement.GetInt32() : 300, + SilenceDurationInMilliseconds = turnDetectionElement.TryGetProperty("silence_duration_ms", out var sdElement) && sdElement.ValueKind == JsonValueKind.Number ? sdElement.GetInt32() : 500, + Threshold = turnDetectionElement.TryGetProperty("threshold", out var thElement) && thElement.ValueKind == JsonValueKind.Number ? thElement.GetDouble() : 0.5, + }; + } + + if (vadType == "semantic_vad") + { + return new SemanticVoiceActivityDetection + { + CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), + InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), + Eagerness = turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && eagernessElement.GetString() is string eagerness + ? new SemanticEagerness(eagerness) + : SemanticEagerness.Auto, + }; + } + + return null; } private static RealtimeServerErrorMessage? CreateErrorMessage(JsonElement root) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 87ca8f7efdf..899843fbaf1 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -341,7 +341,7 @@ private static bool ExtractFunctionCalls(RealtimeServerResponseOutputItemMessage /// private bool ShouldTerminateBasedOnFunctionCalls(List functionCallContents) { - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); if (toolMap is null || toolMap.Count == 0) { @@ -389,7 +389,7 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct CancellationToken cancellationToken) { // Compute toolMap to ensure we always use the latest tools - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); var captureCurrentIterationExceptions = consecutiveErrorCount < MaximumConsecutiveErrorsPerRequest; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index c7ed4204248..2e50aba62c4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -34,8 +34,6 @@ public void Constructor_Parameterless_PropsDefaulted() [Fact] public void Properties_Roundtrip() { - RealtimeSessionOptions options = new(); - var inputFormat = new RealtimeAudioFormat("audio/pcm", 16000); var outputFormat = new RealtimeAudioFormat("audio/pcm", 24000); List modalities = ["text", "audio"]; @@ -43,20 +41,23 @@ public void Properties_Roundtrip() var transcriptionOptions = new TranscriptionOptions { SpeechLanguage = "en", ModelId = "whisper-1", Prompt = "greeting" }; var vad = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; - options.SessionKind = RealtimeSessionKind.Transcription; - options.Model = "gpt-4-realtime"; - options.InputAudioFormat = inputFormat; - options.OutputAudioFormat = outputFormat; - options.NoiseReductionOptions = NoiseReductionOptions.NearField; - options.TranscriptionOptions = transcriptionOptions; - options.VoiceActivityDetection = vad; - options.VoiceSpeed = 1.5; - options.Voice = "alloy"; - options.Instructions = "Be helpful"; - options.MaxOutputTokens = 500; - options.OutputModalities = modalities; - options.ToolMode = ChatToolMode.Auto; - options.Tools = tools; + RealtimeSessionOptions options = new() + { + SessionKind = RealtimeSessionKind.Transcription, + Model = "gpt-4-realtime", + InputAudioFormat = inputFormat, + OutputAudioFormat = outputFormat, + NoiseReductionOptions = NoiseReductionOptions.NearField, + TranscriptionOptions = transcriptionOptions, + VoiceActivityDetection = vad, + VoiceSpeed = 1.5, + Voice = "alloy", + Instructions = "Be helpful", + MaxOutputTokens = 500, + OutputModalities = modalities, + ToolMode = ChatToolMode.Auto, + Tools = tools, + }; Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); Assert.Equal("gpt-4-realtime", options.Model); @@ -107,10 +108,9 @@ public void VoiceActivityDetection_Properties_Roundtrip() Assert.False(vad.CreateResponse); Assert.False(vad.InterruptResponse); - vad.CreateResponse = true; - vad.InterruptResponse = true; + var vad2 = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; - Assert.True(vad.CreateResponse); - Assert.True(vad.InterruptResponse); + Assert.True(vad2.CreateResponse); + Assert.True(vad2.InterruptResponse); } } From aa17989732890702ed3f6873416f4e7c01dfbcec Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 13:01:52 -0800 Subject: [PATCH 38/38] Fix XML doc param order, add null validation to constructors --- .../Realtime/RealtimeClientConversationItemCreateMessage.cs | 3 ++- .../Realtime/RealtimeClientInputAudioBufferAppendMessage.cs | 3 ++- .../Realtime/RealtimeContentItem.cs | 2 +- .../Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs index 979e76f687b..dc10766e347 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -20,7 +21,7 @@ public class RealtimeClientConversationItemCreateMessage : RealtimeClientMessage public RealtimeClientConversationItemCreateMessage(RealtimeContentItem item, string? previousId = null) { PreviousId = previousId; - Item = item; + Item = Throw.IfNull(item); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index 2e1f9c998d2..b1b9dd2f038 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -19,7 +20,7 @@ public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage /// The data content containing the audio buffer data to append. public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) { - Content = audioContent; + Content = Throw.IfNull(audioContent); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs index 9f3040dc326..f6d6d7e9c39 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -20,9 +20,9 @@ public class RealtimeContentItem /// /// Initializes a new instance of the class. /// + /// The contents of the conversation item. /// The ID of the conversation item. /// The role of the conversation item. - /// The contents of the conversation item. public RealtimeContentItem(IList contents, string? id = null, ChatRole? role = null) { Id = id; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 52571623ead..520b117b0a3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -64,8 +64,8 @@ public sealed class OpenAIRealtimeSession : IRealtimeSession /// The model to use for the session. public OpenAIRealtimeSession(string apiKey, string model) { - _apiKey = apiKey; - _model = model; + _apiKey = Throw.IfNull(apiKey); + _model = Throw.IfNull(model); _eventChannel = Channel.CreateUnbounded(); }