diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/DelegatingHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/DelegatingHostedFileClient.cs new file mode 100644 index 00000000000..4c2a809ac33 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/DelegatingHostedFileClient.cs @@ -0,0 +1,102 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating file client that wraps an inner . +/// +/// +/// This class provides a base for creating file clients that modify or enhance the behavior +/// of another . By default, all methods delegate to the inner client. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public class DelegatingHostedFileClient : IHostedFileClient +{ + /// + /// Initializes a new instance of the class. + /// + /// The inner client to delegate to. + /// is . + protected DelegatingHostedFileClient(IHostedFileClient innerClient) + { + InnerClient = Throw.IfNull(innerClient); + } + + /// Gets the inner . + protected IHostedFileClient InnerClient { get; } + + /// + public virtual Task UploadAsync( + Stream content, + string? mediaType = null, + string? fileName = null, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default) => + InnerClient.UploadAsync(content, mediaType, fileName, options, cancellationToken); + + /// + public virtual Task DownloadAsync( + string fileId, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) => + InnerClient.DownloadAsync(fileId, options, cancellationToken); + + /// + public virtual Task GetFileInfoAsync( + string fileId, + HostedFileGetOptions? options = null, + CancellationToken cancellationToken = default) => + InnerClient.GetFileInfoAsync(fileId, options, cancellationToken); + + /// + public virtual IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, + CancellationToken cancellationToken = default) => + InnerClient.ListFilesAsync(options, cancellationToken); + + /// + public virtual Task DeleteAsync( + string fileId, + HostedFileDeleteOptions? options = null, + CancellationToken cancellationToken = default) => + InnerClient.DeleteAsync(fileId, options, cancellationToken); + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerClient.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the instance. + /// + /// + /// if being called from ; otherwise, . + /// + protected virtual void Dispose(bool disposing) + { + // By default, do not dispose the inner client, as it may be shared. + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFile.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFile.cs new file mode 100644 index 00000000000..3aac4ad29a1 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFile.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents metadata about a file hosted by an AI service. +/// +[DebuggerDisplay("{DebuggerDisplay,nq}")] +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFile +{ + /// + /// Initializes a new instance of the class. + /// + /// The unique identifier of the file. + /// is . + /// is empty or composed entirely of whitespace. + public HostedFile(string id) + { + Id = Throw.IfNullOrWhitespace(id); + } + + /// Gets the unique identifier of the file. + public string Id { get; } + + /// Gets or sets the name of the file. + public string? Name { get; set; } + + /// Gets or sets the media type (MIME type) of the file. + public string? MediaType { get; set; } + + /// Gets or sets the size of the file in bytes. + public long? SizeInBytes { get; set; } + + /// Gets or sets when the file was created. + public DateTimeOffset? CreatedAt { get; set; } + + /// Gets or sets the purpose for which the file was uploaded. + /// + /// Common values include "assistants", "fine-tune", "batch", or "vision", + /// but the specific values supported depend on the provider. + /// + public string? Purpose { get; set; } + + /// Gets or sets additional properties associated with the file. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } + + /// Gets or sets the raw representation of the file from the underlying provider. + /// + /// If the was created from an underlying provider's response, + /// this property contains the original response object. + /// + public object? RawRepresentation { get; set; } + + /// + /// Creates a that references this file. + /// + /// A new instance referencing this file. + public HostedFileContent ToHostedFileContent() => + new(Id) + { + Name = Name, + MediaType = MediaType + }; + + /// Gets a string representing this instance to display in the debugger. + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + private string DebuggerDisplay + { + get + { + string display = $"Id = {Id}"; + + if (Name is not null) + { + display += $", Name = \"{Name}\""; + } + + if (MediaType is not null) + { + display += $", MediaType = {MediaType}"; + } + + if (SizeInBytes is not null) + { + display += $", Size = {SizeInBytes} bytes"; + } + + return display; + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientExtensions.cs new file mode 100644 index 00000000000..d572ea757e6 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientExtensions.cs @@ -0,0 +1,253 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net.Mime; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Extension methods for . +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public static class HostedFileClientExtensions +{ + /// + /// Uploads content from a . + /// + /// The file client. + /// The content to upload. + /// Options to configure the upload. + /// The to monitor for cancellation requests. + /// Information about the uploaded file. + /// is . + /// is . + public static Task UploadAsync( + this IHostedFileClient client, + DataContent content, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(content); + + MemoryStream stream = MemoryMarshal.TryGetArray(content.Data, out ArraySegment arraySegment) ? + new(arraySegment.Array!, arraySegment.Offset, arraySegment.Count) : + new(content.Data.ToArray()); + + return client.UploadAsync(stream, content.MediaType, content.Name, options, cancellationToken); + } + + /// + /// Uploads a file from a local file path. + /// + /// The file client. + /// The path to the file to upload. + /// Options to configure the upload. + /// The to monitor for cancellation requests. + /// Information about the uploaded file. + /// is . + /// is . + /// is empty. + public static async Task UploadAsync( + this IHostedFileClient client, + string filePath, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNullOrEmpty(filePath); + + string? mediaType = MediaTypeMap.GetMediaType(filePath); + string fileName = Path.GetFileName(filePath); + + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true); + + return await client.UploadAsync(stream, mediaType, fileName, options, cancellationToken).ConfigureAwait(false); + } + + /// + /// Downloads a file and saves it to a local path. + /// + /// The file client. + /// The ID of the file to download. + /// + /// The path to save the file to. If the path is a directory, the file name will be inferred. + /// + /// Options to configure the download. + /// The to monitor for cancellation requests. + /// The actual path where the file was saved. + /// is . + /// is . + /// is empty or whitespace. + /// is . + public static async Task DownloadToAsync( + this IHostedFileClient client, + string fileId, + string destinationPath, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNullOrWhitespace(fileId); + _ = Throw.IfNull(destinationPath); + + using HostedFileDownloadStream downloadStream = await client.DownloadAsync(fileId, options, cancellationToken).ConfigureAwait(false); + + // Determine the final path + string finalPath = destinationPath; + if (Directory.Exists(destinationPath)) + { + string fileName = downloadStream.FileName ?? fileId; + finalPath = Path.Combine(destinationPath, fileName); + } + + using FileStream fileStream = new(finalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true); + + await downloadStream.CopyToAsync(fileStream, +#if !NET + 81920, +#endif + cancellationToken).ConfigureAwait(false); + + return finalPath; + } + + /// + /// Downloads a file referenced by a . + /// + /// The file client. + /// The hosted file reference. + /// Options to configure the download. + /// The to monitor for cancellation requests. + /// A containing the file content. + /// is . + /// is . + public static Task DownloadAsync( + this IHostedFileClient client, + HostedFileContent hostedFile, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(hostedFile); + + return client.DownloadAsync(hostedFile.FileId, options, cancellationToken); + } + + /// + /// Downloads a file and returns its content as a buffered . + /// + /// The file client. + /// The ID of the file to download. + /// Options to configure the download. + /// The to monitor for cancellation requests. + /// The file content as a . + /// is . + /// is . + /// is empty or whitespace. + /// + /// This method buffers the entire file content into memory. For large files, + /// consider using + /// and streaming directly to the destination. + /// + public static async Task DownloadAsDataContentAsync( + this IHostedFileClient client, + string fileId, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNullOrWhitespace(fileId); + + using HostedFileDownloadStream downloadStream = await client.DownloadAsync(fileId, options, cancellationToken).ConfigureAwait(false); + + return await downloadStream.ToDataContentAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Gets the for this client. + /// + /// The file client. + /// The metadata for this client, or if not available. + /// is . + public static HostedFileClientMetadata? GetMetadata(this IHostedFileClient client) + { + _ = Throw.IfNull(client); + + return client.GetService(typeof(HostedFileClientMetadata)) as HostedFileClientMetadata; + } + + /// + /// Gets a service of the specified type from the file client. + /// + /// The type of service to retrieve. + /// The file client. + /// An optional key that can be used to help identify the target service. + /// The found service, or if not available. + /// is . + public static TService? GetService(this IHostedFileClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + return client.GetService(typeof(TService), serviceKey) is TService service ? service : default; + } + + /// + /// Asks the for an object of the specified type + /// and throws an exception if one isn't available. + /// + /// The file client. + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static object GetRequiredService(this IHostedFileClient client, Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(serviceType); + + return + client.GetService(serviceType, serviceKey) ?? + throw Throw.CreateMissingServiceException(serviceType, serviceKey); + } + + /// + /// Asks the for an object of type + /// and throws an exception if one isn't available. + /// + /// The type of the object to be retrieved. + /// The file client. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static TService GetRequiredService(this IHostedFileClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + if (client.GetService(typeof(TService), serviceKey) is not TService service) + { + throw Throw.CreateMissingServiceException(typeof(TService), serviceKey); + } + + return service; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientMetadata.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientMetadata.cs new file mode 100644 index 00000000000..9279ef07d48 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientMetadata.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides metadata about an . +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public class HostedFileClientMetadata +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the file client provider, if applicable. + /// The URI of the provider's endpoint, if applicable. + public HostedFileClientMetadata(string? providerName = null, Uri? providerUri = null) + { + ProviderName = providerName; + ProviderUri = providerUri; + } + + /// Gets the name of the file client provider. + /// + /// Where possible, this maps to the name of the company or organization that provides the + /// underlying file storage, such as "openai", "anthropic", or "google". + /// + public string? ProviderName { get; } + + /// Gets the URI of the provider's endpoint. + public Uri? ProviderUri { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientOptions.cs new file mode 100644 index 00000000000..8b3a24e6d95 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileClientOptions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Base class for options used with operations. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public abstract class HostedFileClientOptions +{ + /// + /// Initializes a new instance of the class. + /// + private protected HostedFileClientOptions() + { + // Prevent external derivation + } + + /// + /// Gets or sets a provider-specific scope or location identifier for the file operation. + /// + /// + /// Some providers use scoped storage for files. For example, OpenAI uses containers + /// to scope code interpreter files. If specified, the operation will target + /// files within the specified scope. + /// + public string? Scope { get; set; } + + /// Gets or sets additional properties for the request. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDeleteOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDeleteOptions.cs new file mode 100644 index 00000000000..e384195e4af --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDeleteOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Options for deleting a file from an AI service. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileDeleteOptions : HostedFileClientOptions; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadOptions.cs new file mode 100644 index 00000000000..766b9ddae70 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Options for downloading a file from an AI service. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileDownloadOptions : HostedFileClientOptions; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadStream.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadStream.cs new file mode 100644 index 00000000000..6ed66ea8284 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadStream.cs @@ -0,0 +1,80 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a stream for downloading file content from an AI service. +/// +/// +/// +/// This abstract class extends to provide additional metadata +/// about the downloaded file, such as its media type and file name. Implementations +/// should override the abstract members and optionally override +/// and to provide file metadata. +/// +/// +/// The method provides a convenient way to buffer +/// the entire stream content into a instance. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public abstract class HostedFileDownloadStream : Stream +{ + /// + /// Initializes a new instance of the class. + /// + protected HostedFileDownloadStream() + { + } + + /// + /// Gets the media type (MIME type) of the file content. + /// + /// + /// Returns if the media type is not known. + /// + public virtual string? MediaType => null; + + /// + /// Gets the file name. + /// + /// + /// Returns if the file name is not known. + /// + public virtual string? FileName => null; + + /// + /// Reads the entire stream content and returns it as a . + /// + /// The to monitor for cancellation requests. + /// A containing the buffered file content. + /// + /// This method buffers the entire stream content into memory. For large files, + /// consider streaming directly to the destination instead. + /// + public virtual async Task ToDataContentAsync(CancellationToken cancellationToken = default) + { + MemoryStream memoryStream = new(); + + await CopyToAsync(memoryStream, +#if !NET + 81920, +#endif + cancellationToken).ConfigureAwait(false); + + return new DataContent( + memoryStream.GetBuffer().AsMemory(0, (int)memoryStream.Length), + MediaType ?? "application/octet-stream") + { + Name = FileName, + }; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileGetOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileGetOptions.cs new file mode 100644 index 00000000000..5c0231549d1 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileGetOptions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Options for getting file metadata from an AI service. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileGetOptions : HostedFileClientOptions; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileListOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileListOptions.cs new file mode 100644 index 00000000000..77430c17d24 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileListOptions.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Options for listing files from an AI service. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileListOptions : HostedFileClientOptions +{ + /// Gets or sets a purpose filter. + /// + /// If specified, only files with the given purpose will be returned. + /// + public string? Purpose { get; set; } + + /// Gets or sets the maximum number of files to return. + /// + /// If not specified, the provider's default limit will be used. + /// + public int? Limit { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileUploadOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileUploadOptions.cs new file mode 100644 index 00000000000..d77eded60f5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileUploadOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Options for uploading a file to an AI service. +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileUploadOptions : HostedFileClientOptions +{ + /// Gets or sets the purpose of the file. + /// + /// + /// If not specified, implementations may default to a provider-specific value + /// (typically "assistants" or equivalent for code interpreter use). + /// + /// + /// Common values include "assistants", "fine-tune", "batch", and "vision", + /// but the specific values supported depend on the provider. + /// + /// + public string? Purpose { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/IHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/IHostedFileClient.cs new file mode 100644 index 00000000000..f26b0e4927b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/IHostedFileClient.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a client for uploading, downloading, and managing files hosted by an AI service. +/// +/// +/// +/// File clients enable interaction with server-side file storage used by AI services, +/// particularly for code interpreter inputs and outputs. Files uploaded through this +/// interface can be referenced in AI requests using . +/// +/// +/// Unless otherwise specified, all members of are thread-safe +/// for concurrent use. It is expected that all implementations of +/// support being used by multiple requests concurrently. Instances must not be disposed +/// of while the instance is still in use. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public interface IHostedFileClient : IDisposable +{ + /// + /// Uploads a file to the AI service. + /// + /// The stream containing the file content to upload. + /// The media type (MIME type) of the content. + /// The name of the file. + /// Options to configure the upload. + /// The to monitor for cancellation requests. + /// Information about the uploaded file. + /// is . + Task UploadAsync( + Stream content, + string? mediaType = null, + string? fileName = null, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default); + + /// + /// Downloads a file from the AI service. + /// + /// The ID of the file to download. + /// Options to configure the download. + /// The to monitor for cancellation requests. + /// + /// A containing the file content. The stream should be disposed when no longer needed. + /// + /// is . + /// is empty or whitespace. + Task DownloadAsync( + string fileId, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default); + + /// + /// Gets metadata about a file. + /// + /// The ID of the file. + /// Options to configure the request. + /// The to monitor for cancellation requests. + /// Information about the file, or if not found. + /// is . + /// is empty or whitespace. + Task GetFileInfoAsync( + string fileId, + HostedFileGetOptions? options = null, + CancellationToken cancellationToken = default); + + /// + /// Lists files accessible by this client. + /// + /// Options to configure the listing. + /// The to monitor for cancellation requests. + /// An async enumerable of file information. + IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, + CancellationToken cancellationToken = default); + + /// + /// Deletes a file from the AI service. + /// + /// The ID of the file to delete. + /// Options to configure the request. + /// The to monitor for cancellation requests. + /// if the file was deleted; if the file was not found. + /// is . + /// is empty or whitespace. + Task DeleteAsync( + string fileId, + HostedFileDeleteOptions? options = null, + CancellationToken cancellationToken = default); + + /// + /// Asks the for an object of the specified type . + /// + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 0f2e4340358..cd9a4dadf04 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -162,6 +162,15 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(ImageGenerationOptions))] [JsonSerializable(typeof(ImageGenerationResponse))] + // IHostedFileClient + [JsonSerializable(typeof(HostedFile))] + [JsonSerializable(typeof(HostedFileUploadOptions))] + [JsonSerializable(typeof(HostedFileDownloadOptions))] + [JsonSerializable(typeof(HostedFileGetOptions))] + [JsonSerializable(typeof(HostedFileListOptions))] + [JsonSerializable(typeof(HostedFileDeleteOptions))] + [JsonSerializable(typeof(HostedFileClientMetadata))] + [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead. private sealed partial class JsonContext : JsonSerializerContext; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs index e4008e4380f..6d223e6d5d7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; @@ -16,7 +16,9 @@ using OpenAI.Assistants; using OpenAI.Audio; using OpenAI.Chat; +using OpenAI.Containers; using OpenAI.Embeddings; +using OpenAI.Files; using OpenAI.Images; using OpenAI.Responses; @@ -180,6 +182,44 @@ public static IImageGenerator AsIImageGenerator(this ImageClient imageClient) => public static IEmbeddingGenerator> AsIEmbeddingGenerator(this EmbeddingClient embeddingClient, int? defaultModelDimensions = null) => new OpenAIEmbeddingGenerator(embeddingClient, defaultModelDimensions); + /// Gets an for use with this . + /// The client. + /// An that can be used to manage files via the . + /// is . + /// + /// The returned supports both the standard Files API and container files + /// (used for code interpreter outputs). To download a container file, specify the container ID + /// in the property. + /// + [Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] + public static IHostedFileClient AsIHostedFileClient(this OpenAIClient openAIClient) => + new OpenAIHostedFileClient(openAIClient); + + /// Gets an for use with this . + /// The client. + /// An that can be used to manage files via the . + /// is . + /// + /// The returned supports only the standard Files API. + /// Operations requiring container access (via Scope) will throw . + /// To access container files, use or . + /// + [Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] + public static IHostedFileClient AsIHostedFileClient(this OpenAIFileClient fileClient) => + new OpenAIHostedFileClient(fileClient); + + /// Gets an for use with this . + /// The client. + /// + /// The default container ID for operations. If not specified, a container ID must be + /// provided via the Scope property on per-call options. + /// + /// An that can be used to manage files within containers via the . + /// is . + [Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] + public static IHostedFileClient AsIHostedFileClient(this ContainerClient containerClient, string? defaultScope = null) => + new OpenAIHostedFileClient(containerClient, defaultScope); + /// Gets whether the properties specify that strict schema handling is desired. internal static bool? HasStrict(IReadOnlyDictionary? additionalProperties) => additionalProperties?.TryGetValue(StrictKey, out object? strictObj) is true && diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIFileDownloadStream.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIFileDownloadStream.cs new file mode 100644 index 00000000000..bca8ccd3407 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIFileDownloadStream.cs @@ -0,0 +1,133 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +#pragma warning disable S4136 // Method overloads should be grouped together - .NET Core overloads grouped in #if NET block below + +namespace Microsoft.Extensions.AI; + +/// +/// A implementation for OpenAI file downloads. +/// +internal sealed class OpenAIFileDownloadStream : HostedFileDownloadStream +{ + private readonly Stream _innerStream; + + /// + /// Initializes a new instance of the class. + /// + /// The downloaded file data. + /// The media type of the file. + /// The file name. + public OpenAIFileDownloadStream(BinaryData data, string? mediaType, string? fileName) + { + _innerStream = data.ToStream(); + MediaType = mediaType; + FileName = fileName; + } + + /// + public override string? MediaType { get; } + + /// + public override string? FileName { get; } + + /// + public override bool CanRead => _innerStream.CanRead; + + /// + public override bool CanSeek => _innerStream.CanSeek; + + /// + public override bool CanWrite => false; + + /// + public override long Length => _innerStream.Length; + + /// + public override long Position + { + get => _innerStream.Position; + set => _innerStream.Position = value; + } + + /// + public override void Flush() => + _innerStream.Flush(); + + /// + public override Task FlushAsync(CancellationToken cancellationToken) => + _innerStream.FlushAsync(cancellationToken); + + /// + public override int ReadByte() => + _innerStream.ReadByte(); + + /// + public override int Read(byte[] buffer, int offset, int count) => + _innerStream.Read(buffer, offset, count); + + /// + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + + /// + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + /// + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + /// + public override long Seek(long offset, SeekOrigin origin) => + _innerStream.Seek(offset, origin); + + /// + public override void SetLength(long value) => + _innerStream.SetLength(value); + + /// + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) => + _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _innerStream.Dispose(); + } + + base.Dispose(disposing); + } + +#if NET + public override void CopyTo(Stream destination, int bufferSize) => _innerStream.CopyTo(destination, bufferSize); + + /// + public override int Read(Span buffer) => _innerStream.Read(buffer); + + /// + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + _innerStream.ReadAsync(buffer, cancellationToken); + + /// + public override void Write(ReadOnlySpan buffer) => throw new NotSupportedException(); + + /// + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + /// + public override async ValueTask DisposeAsync() + { + await _innerStream.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } +#endif +} diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIHostedFileClient.cs new file mode 100644 index 00000000000..6d41bd8275d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIHostedFileClient.cs @@ -0,0 +1,405 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; +using OpenAI; +using OpenAI.Containers; +using OpenAI.Files; + +#pragma warning disable CA1031 // Do not catch general exception types +#pragma warning disable IDE0058 // Expression value is never used + +namespace Microsoft.Extensions.AI; + +/// +/// An implementation for OpenAI file operations. +/// +/// +/// +/// This client supports both the standard Files API and container-scoped files (used for code interpreter outputs). +/// When a (container ID) is specified on a per-call options object +/// or as the default scope at construction time, operations target that container. Otherwise, operations use +/// the standard Files API. +/// +/// +/// Depending on how this client is constructed, it may support only file operations, only container operations, +/// or both. If an operation requires a client that was not provided, an is thrown. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class OpenAIHostedFileClient : IHostedFileClient +{ + /// The underlying for standard file operations, or if not available. + private readonly OpenAIFileClient? _fileClient; + + /// The underlying for container file operations, or if not available. + private readonly ContainerClient? _containerClient; + + /// The default scope (container ID) for operations, or if not set. + private readonly string? _defaultScope; + + /// The metadata for this client. + private readonly HostedFileClientMetadata _metadata; + + /// + /// Initializes a new instance of the class from an . + /// + /// The underlying . + public OpenAIHostedFileClient(OpenAIClient openAIClient) + { + _ = Throw.IfNull(openAIClient); + _fileClient = openAIClient.GetOpenAIFileClient(); + _containerClient = openAIClient.GetContainerClient(); + _metadata = new HostedFileClientMetadata("openai", _fileClient.Endpoint); + } + + /// + /// Initializes a new instance of the class from an . + /// + /// The underlying . + public OpenAIHostedFileClient(OpenAIFileClient fileClient) + { + _fileClient = Throw.IfNull(fileClient); + _metadata = new HostedFileClientMetadata("openai", _fileClient.Endpoint); + } + + /// + /// Initializes a new instance of the class from a . + /// + /// The underlying . + /// + /// The default container ID for operations. If not specified, a container ID must be + /// provided via the property on per-call options. + /// + public OpenAIHostedFileClient(ContainerClient containerClient, string? defaultScope = null) + { + _containerClient = Throw.IfNull(containerClient); + _defaultScope = defaultScope; + _metadata = new HostedFileClientMetadata("openai", _containerClient.Endpoint); + } + + /// + public async Task UploadAsync( + Stream content, + string? mediaType = null, + string? fileName = null, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(content); + + fileName ??= content is FileStream fs ? Path.GetFileName(fs.Name) : null; + mediaType ??= fileName is not null ? MediaTypeMap.GetMediaType(fileName) : null; + fileName ??= $"{Guid.NewGuid():N}{MediaTypeMap.GetExtension(mediaType)}"; + + if (ResolveScope(options) is string containerId) + { + mediaType ??= "application/octet-stream"; + + using MultipartFormDataContent multipart = new(); + using NonDisposingStreamContent fileContent = new(content); + fileContent.Headers.ContentType = new MediaTypeHeaderValue(mediaType); + multipart.Add(fileContent, "file", fileName); + + using var binaryContent = new HttpContentBinaryContent(multipart); + + var result = await GetContainerClient().CreateContainerFileAsync( + containerId, + binaryContent, + multipart.Headers.ContentType!.ToString(), + new RequestOptions + { + CancellationToken = cancellationToken + }).ConfigureAwait(false); + + using var responseDoc = JsonDocument.Parse(result.GetRawResponse().Content); + var root = responseDoc.RootElement; + + string fileId = root.GetProperty("id").GetString()!; + string? path = root.TryGetProperty("path", out var pathProp) ? pathProp.GetString() : null; + string name = path is not null ? Path.GetFileName(path) : fileName; + + return new HostedFile(fileId) + { + Name = name, + MediaType = MediaTypeMap.GetMediaType(name), + }; + } + else + { + var purpose = options?.Purpose switch + { + "assistants" or null => FileUploadPurpose.Assistants, + "fine-tune" => FileUploadPurpose.FineTune, + "batch" => FileUploadPurpose.Batch, + "vision" => FileUploadPurpose.Vision, + _ => new FileUploadPurpose(options.Purpose) + }; + + var result = await GetFileClient().UploadFileAsync(content, fileName, purpose, cancellationToken).ConfigureAwait(false); + + return ToHostedFile(result.Value); + } + } + + /// + public async Task DownloadAsync( + string fileId, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNullOrWhitespace(fileId); + + if (ResolveScope(options) is string containerId) + { + var containerClient = GetContainerClient(); + var containerResult = await containerClient.DownloadContainerFileAsync(containerId, fileId, cancellationToken).ConfigureAwait(false); + var containerFileInfo = await containerClient.GetContainerFileAsync(containerId, fileId, cancellationToken).ConfigureAwait(false); + + string containerFileName = Path.GetFileName(containerFileInfo.Value.Path); + string? containerMediaType = MediaTypeMap.GetMediaType(containerFileName) ?? "application/octet-stream"; + + return new OpenAIFileDownloadStream(containerResult.Value, containerMediaType, containerFileName); + } + else + { + var fileClient = GetFileClient(); + var result = await fileClient.DownloadFileAsync(fileId, cancellationToken).ConfigureAwait(false); + var fileInfo = await fileClient.GetFileAsync(fileId, cancellationToken).ConfigureAwait(false); + + string? mediaType = MediaTypeMap.GetMediaType(fileInfo.Value.Filename) ?? "application/octet-stream"; + + return new OpenAIFileDownloadStream(result.Value, mediaType, fileInfo.Value.Filename); + } + } + + /// + public async Task GetFileInfoAsync( + string fileId, + HostedFileGetOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNullOrWhitespace(fileId); + + try + { + if (ResolveScope(options) is string containerId) + { + var containerResult = await GetContainerClient().GetContainerFileAsync( + containerId, fileId, cancellationToken).ConfigureAwait(false); + + var containerFile = containerResult.Value; + string name = Path.GetFileName(containerFile.Path); + + return new HostedFile(containerFile.Id) + { + Name = name, + MediaType = MediaTypeMap.GetMediaType(name), + RawRepresentation = containerFile + }; + } + else + { + var result = await GetFileClient().GetFileAsync(fileId, cancellationToken).ConfigureAwait(false); + return ToHostedFile(result.Value); + } + } + catch (Exception ex) when (IsNotFoundError(ex)) + { + return null; + } + } + + /// + public async IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + int limit = options?.Limit ?? int.MaxValue; + + if (ResolveScope(options) is string containerId) + { + int count = 0; + await foreach (var containerFile in GetContainerClient().GetContainerFilesAsync(containerId, cancellationToken: cancellationToken).ConfigureAwait(false)) + { + if (count >= limit) + { + yield break; + } + + string name = Path.GetFileName(containerFile.Path); + + yield return new HostedFile(containerFile.Id) + { + Name = name, + MediaType = MediaTypeMap.GetMediaType(name), + RawRepresentation = containerFile + }; + + count++; + } + } + else + { + FilePurpose? purpose = options?.Purpose switch + { + "assistants" => FilePurpose.Assistants, + "fine-tune" => FilePurpose.FineTune, + "batch" => FilePurpose.Batch, + "vision" => FilePurpose.Vision, + _ => null + }; + + var fileClient = GetFileClient(); + var result = await (purpose is FilePurpose p ? + fileClient.GetFilesAsync(p, cancellationToken) : + fileClient.GetFilesAsync(cancellationToken)).ConfigureAwait(false); + + int count = 0; + foreach (var file in result.Value) + { + if (count >= limit) + { + yield break; + } + + yield return ToHostedFile(file); + count++; + } + } + } + + /// + public async Task DeleteAsync( + string fileId, + HostedFileDeleteOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNullOrWhitespace(fileId); + + try + { + if (ResolveScope(options) is string containerId) + { + await GetContainerClient().DeleteContainerFileAsync(containerId, fileId, cancellationToken).ConfigureAwait(false); + return true; + } + else + { + var result = await GetFileClient().DeleteFileAsync(fileId, cancellationToken).ConfigureAwait(false); + return result.Value.Deleted; + } + } + catch (Exception ex) when (IsNotFoundError(ex)) + { + return false; + } + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(HostedFileClientMetadata) ? _metadata : + serviceType == typeof(OpenAIFileClient) ? _fileClient : + serviceType == typeof(ContainerClient) ? _containerClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + public void Dispose() + { + // Nothing to dispose; the underlying clients are not owned by this instance. + } + + private static HostedFile ToHostedFile(OpenAIFile openAIFile) => + new(openAIFile.Id) + { + Name = openAIFile.Filename, + SizeInBytes = openAIFile.SizeInBytes, + CreatedAt = openAIFile.CreatedAt, + Purpose = openAIFile.Purpose.ToString(), + MediaType = MediaTypeMap.GetMediaType(openAIFile.Filename), + RawRepresentation = openAIFile, + }; + + private static bool IsNotFoundError(Exception ex) => + ex is ClientResultException { Status: 404 }; + + private OpenAIFileClient GetFileClient() => + _fileClient ?? + throw new InvalidOperationException( + $"This operation requires the standard Files API, but this client was not constructed with an {nameof(OpenAIFileClient)}. " + + $"Use an {nameof(IHostedFileClient)} created from an {nameof(OpenAIClient)} or {nameof(OpenAIFileClient)}, or set the Scope option to target a container instead."); + + private ContainerClient GetContainerClient() => + _containerClient ?? + throw new InvalidOperationException( + $"This operation requires a container (Scope was specified), but this client was not constructed with a {nameof(ContainerClient)}. " + + $"Use an {nameof(IHostedFileClient)} created from an {nameof(OpenAIClient)} or {nameof(ContainerClient)} to access container files."); + + /// Resolves the scope (container ID) from per-call options or the default. + private string? ResolveScope(HostedFileClientOptions? options) => + options?.Scope ?? _defaultScope; + + /// A that writes an directly to the output stream. + private sealed class HttpContentBinaryContent(HttpContent httpContent) : BinaryContent + { + public override void WriteTo(Stream stream, CancellationToken cancellationToken = default) + { +#if NET + httpContent.CopyTo(stream, null, cancellationToken); +#else +#pragma warning disable VSTHRD002 // Synchronously waiting - no sync CopyTo on older TFMs + httpContent.CopyToAsync(stream).GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 +#endif + } + + public override Task WriteToAsync(Stream stream, CancellationToken cancellationToken = default) => +#if NET + httpContent.CopyToAsync(stream, cancellationToken); +#else + httpContent.CopyToAsync(stream); +#endif + + public override bool TryComputeLength(out long length) + { + length = httpContent.Headers.ContentLength.GetValueOrDefault(-1); + return length >= 0; + } + + public override void Dispose() + { + } + } + + /// A that does not dispose the underlying stream. + private sealed class NonDisposingStreamContent(Stream stream) : StreamContent(stream) + { +#pragma warning disable CA2215 // Intentionally not calling base.Dispose to avoid disposing the caller's stream + protected override void Dispose(bool disposing) + { + // Do not call base.Dispose; it would dispose the caller's stream. + } +#pragma warning restore CA2215 + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilder.cs new file mode 100644 index 00000000000..1a8ecc159da --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilder.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A builder for creating pipelines of . +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class HostedFileClientBuilder +{ + private readonly Func _innerClientFactory; + + /// The registered client factory instances. + private List>? _clientFactories; + + /// Initializes a new instance of the class. + /// The inner that represents the underlying backend. + public HostedFileClientBuilder(IHostedFileClient innerClient) + { + _ = Throw.IfNull(innerClient); + _innerClientFactory = _ => innerClient; + } + + /// Initializes a new instance of the class. + /// A callback that produces the inner that represents the underlying backend. + public HostedFileClientBuilder(Func innerClientFactory) + { + _innerClientFactory = Throw.IfNull(innerClientFactory); + } + + /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. + /// + /// The that should provide services to the instances. + /// If null, an empty will be used. + /// + /// An instance of that represents the entire pipeline. + public IHostedFileClient Build(IServiceProvider? services = null) + { + services ??= EmptyServiceProvider.Instance; + var fileClient = _innerClientFactory(services); + + // To match intuitive expectations, apply the factories in reverse order, so that the first factory added is the outermost. + if (_clientFactories is not null) + { + for (var i = _clientFactories.Count - 1; i >= 0; i--) + { + fileClient = _clientFactories[i](fileClient, services) ?? + throw new InvalidOperationException( + $"The {nameof(HostedFileClientBuilder)} entry at index {i} returned null. " + + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IHostedFileClient)} instances."); + } + } + + return fileClient; + } + + /// Adds a factory for an intermediate hosted file client to the hosted file client pipeline. + /// The client factory function. + /// The updated instance. + public HostedFileClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + return Use((innerClient, _) => clientFactory(innerClient)); + } + + /// Adds a factory for an intermediate hosted file client to the hosted file client pipeline. + /// The client factory function. + /// The updated instance. + public HostedFileClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + (_clientFactories ??= []).Add(clientFactory); + return this; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilderHostedFileClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilderHostedFileClientExtensions.cs new file mode 100644 index 00000000000..acd16776f7e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/HostedFileClientBuilderHostedFileClientExtensions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extension methods for working with in the context of . +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public static class HostedFileClientBuilderHostedFileClientExtensions +{ + /// Creates a new using as its inner client. + /// The client to use as the inner client. + /// The new instance. + /// + /// This method is equivalent to using the constructor directly, + /// specifying as the inner client. + /// + public static HostedFileClientBuilder AsBuilder(this IHostedFileClient innerClient) + { + _ = Throw.IfNull(innerClient); + + return new HostedFileClientBuilder(innerClient); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClient.cs new file mode 100644 index 00000000000..bb3010c434d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClient.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net.Mime; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A delegating hosted file client that logs file operations to an . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// employed is also thread-safe for concurrent use. +/// +/// +/// When the employed enables , the contents of +/// options and results are logged. These may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Options and results are not logged at other logging levels. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed partial class LoggingHostedFileClient : DelegatingHostedFileClient +{ + /// An instance used for all logging. + private readonly ILogger _logger; + + /// The to use for serialization of state written to the logger. + private JsonSerializerOptions _jsonSerializerOptions; + + /// Initializes a new instance of the class. + /// The underlying . + /// An instance that will be used for all logging. + public LoggingHostedFileClient(IHostedFileClient innerClient, ILogger logger) + : base(innerClient) + { + _logger = Throw.IfNull(logger); + _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; + } + + /// Gets or sets JSON serialization options to use when serializing logging data. + public JsonSerializerOptions JsonSerializerOptions + { + get => _jsonSerializerOptions; + set => _jsonSerializerOptions = Throw.IfNull(value); + } + + /// + public override async Task UploadAsync( + Stream content, string? mediaType = null, string? fileName = null, HostedFileUploadOptions? options = null, CancellationToken cancellationToken = default) + { + fileName ??= content is FileStream fs ? Path.GetFileName(fs.Name) : null; + mediaType ??= fileName is not null ? MediaTypeMap.GetMediaType(fileName) : null; + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogUploadInvokedSensitive(mediaType, fileName, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(UploadAsync)); + } + } + + try + { + var result = await base.UploadAsync(content, mediaType, fileName, options, cancellationToken).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogCompletedSensitive(nameof(UploadAsync), AsJson(result)); + } + else + { + LogCompleted(nameof(UploadAsync)); + } + } + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(UploadAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(UploadAsync), ex); + throw; + } + } + + /// + public override async Task DownloadAsync( + string fileId, HostedFileDownloadOptions? options = null, CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogDownloadInvokedSensitive(fileId, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(DownloadAsync)); + } + } + + try + { + var result = await base.DownloadAsync(fileId, options, cancellationToken).ConfigureAwait(false); + + LogCompleted(nameof(DownloadAsync)); + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(DownloadAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(DownloadAsync), ex); + throw; + } + } + + /// + public override async Task GetFileInfoAsync( + string fileId, HostedFileGetOptions? options = null, CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogGetFileInfoInvokedSensitive(fileId, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(GetFileInfoAsync)); + } + } + + try + { + var result = await base.GetFileInfoAsync(fileId, options, cancellationToken).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogCompletedSensitive(nameof(GetFileInfoAsync), AsJson(result)); + } + else + { + LogCompleted(nameof(GetFileInfoAsync)); + } + } + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(GetFileInfoAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(GetFileInfoAsync), ex); + throw; + } + } + + /// + public override async IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogListFilesInvokedSensitive(AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(ListFilesAsync)); + } + } + + IAsyncEnumerator e; + try + { + e = base.ListFilesAsync(options, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ListFilesAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ListFilesAsync), ex); + throw; + } + + try + { + HostedFile? file = null; + while (true) + { + try + { + if (!await e.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + + file = e.Current; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ListFilesAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ListFilesAsync), ex); + throw; + } + + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogListItemSensitive(AsJson(file)); + } + + yield return file; + } + + LogCompleted(nameof(ListFilesAsync)); + } + finally + { + await e.DisposeAsync().ConfigureAwait(false); + } + } + + /// + public override async Task DeleteAsync( + string fileId, HostedFileDeleteOptions? options = null, CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogDeleteInvokedSensitive(fileId, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(DeleteAsync)); + } + } + + try + { + var result = await base.DeleteAsync(fileId, options, cancellationToken).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogCompletedSensitive(nameof(DeleteAsync), AsJson(result)); + } + else + { + LogCompleted(nameof(DeleteAsync)); + } + } + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(DeleteAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(DeleteAsync), ex); + throw; + } + } + + private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] + private partial void LogInvoked(string methodName); + + [LoggerMessage(LogLevel.Trace, "UploadAsync invoked. MediaType: {MediaType}. FileName: {FileName}. Options: {HostedFileOptions}. Metadata: {HostedFileClientMetadata}.")] + private partial void LogUploadInvokedSensitive(string? mediaType, string? fileName, string hostedFileOptions, string hostedFileClientMetadata); + + [LoggerMessage(LogLevel.Trace, "DownloadAsync invoked. FileId: {FileId}. Options: {HostedFileOptions}. Metadata: {HostedFileClientMetadata}.")] + private partial void LogDownloadInvokedSensitive(string fileId, string hostedFileOptions, string hostedFileClientMetadata); + + [LoggerMessage(LogLevel.Trace, "GetFileInfoAsync invoked. FileId: {FileId}. Options: {HostedFileOptions}. Metadata: {HostedFileClientMetadata}.")] + private partial void LogGetFileInfoInvokedSensitive(string fileId, string hostedFileOptions, string hostedFileClientMetadata); + + [LoggerMessage(LogLevel.Trace, "ListFilesAsync invoked. Options: {HostedFileOptions}. Metadata: {HostedFileClientMetadata}.")] + private partial void LogListFilesInvokedSensitive(string hostedFileOptions, string hostedFileClientMetadata); + + [LoggerMessage(LogLevel.Trace, "DeleteAsync invoked. FileId: {FileId}. Options: {HostedFileOptions}. Metadata: {HostedFileClientMetadata}.")] + private partial void LogDeleteInvokedSensitive(string fileId, string hostedFileOptions, string hostedFileClientMetadata); + + [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] + private partial void LogCompleted(string methodName); + + [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {HostedFileResult}.")] + private partial void LogCompletedSensitive(string methodName, string hostedFileResult); + + [LoggerMessage(LogLevel.Trace, "ListFilesAsync received item: {HostedFile}")] + private partial void LogListItemSensitive(string hostedFile); + + [LoggerMessage(LogLevel.Debug, "{MethodName} canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClientBuilderExtensions.cs new file mode 100644 index 00000000000..457460b34f2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/LoggingHostedFileClientBuilderExtensions.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public static class LoggingHostedFileClientBuilderExtensions +{ + /// Adds logging to the hosted file client pipeline. + /// The . + /// + /// An optional used to create a logger with which logging should be performed. + /// If not supplied, a required instance will be resolved from the service provider. + /// + /// An optional callback that can be used to configure the instance. + /// The . + /// + /// + /// When the employed enables , the contents of + /// options and results are logged. These may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Options and results are not logged at other logging levels. + /// + /// + public static HostedFileClientBuilder UseLogging( + this HostedFileClientBuilder builder, + ILoggerFactory? loggerFactory = null, + Action? configure = null) + { + _ = Throw.IfNull(builder); + + return builder.Use((innerClient, services) => + { + loggerFactory ??= services.GetRequiredService(); + + // If the factory we resolve is for the null logger, the LoggingHostedFileClient will end up + // being an expensive nop, so skip adding it and just return the inner client. + if (loggerFactory == NullLoggerFactory.Instance) + { + return innerClient; + } + + var fileClient = new LoggingHostedFileClient(innerClient, loggerFactory.CreateLogger(typeof(LoggingHostedFileClient))); + configure?.Invoke(fileClient); + return fileClient; + }); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs new file mode 100644 index 00000000000..38e68767a86 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClient.cs @@ -0,0 +1,463 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.IO; +using System.Net.Mime; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; + +#pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter +#pragma warning disable SA1113 // Comma should be on the same line as previous parameter + +namespace Microsoft.Extensions.AI; + +/// Represents a delegating hosted file client that implements OpenTelemetry-compatible tracing and metrics for file operations. +/// +/// +/// Since there is currently no OpenTelemetry Semantic Convention for hosted file operations, this implementation +/// uses general client span conventions alongside standard file.* registry attributes where applicable. +/// +/// +/// The specification is subject to change as relevant OpenTelemetry conventions emerge; as such, the telemetry +/// output by this client is also subject to change. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OpenTelemetryHostedFileClient : DelegatingHostedFileClient +{ + private const string UploadOperationName = "files.upload"; + private const string DownloadOperationName = "files.download"; + private const string GetInfoOperationName = "files.get_info"; + private const string ListOperationName = "files.list"; + private const string DeleteOperationName = "files.delete"; + + private const string OperationDurationMetricName = "files.client.operation.duration"; + private const string OperationDurationMetricDescription = "Measures the duration of a file operation"; + + private const string FilesOperationNameAttribute = "files.operation.name"; + private const string FilesProviderNameAttribute = "files.provider.name"; + private const string FilesIdAttribute = "files.id"; + private const string FilesPurposeAttribute = "files.purpose"; + private const string FilesScopeAttribute = "files.scope"; + private const string FilesListCountAttribute = "files.list.count"; + private const string FilesMediaTypeAttribute = "files.media_type"; + + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + + private readonly string? _providerName; + private readonly string? _serverAddress; + private readonly int _serverPort; + + /// Initializes a new instance of the class. + /// The underlying . + /// The to use for emitting any logging data from the client. + /// An optional source name that will be used on the telemetry data. +#pragma warning disable IDE0060 // Remove unused parameter; it exists for consistency with OpenTelemetryChatClient and future use + public OpenTelemetryHostedFileClient(IHostedFileClient innerClient, ILogger? logger = null, string? sourceName = null) +#pragma warning restore IDE0060 + : base(innerClient) + { + Debug.Assert(innerClient is not null, "Should have been validated by the base ctor"); + + if (innerClient!.GetService() is HostedFileClientMetadata metadata) + { + _providerName = metadata.ProviderName; + _serverAddress = metadata.ProviderUri?.Host; + _serverPort = metadata.ProviderUri?.Port ?? 0; + } + + string name = string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!; + _activitySource = new(name); + _meter = new(name); + + _operationDurationHistogram = _meter.CreateHistogram( + OperationDurationMetricName, + OpenTelemetryConsts.SecondsUnit, + OperationDurationMetricDescription, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries } + ); + } + + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is , unless the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + /// environment variable is set to "true" (case-insensitive). + /// + /// + /// + /// By default, telemetry includes operation metadata such as provider name, duration, + /// file IDs, file sizes, media types, purposes, and scopes. + /// + /// + /// When enabled, telemetry will additionally include file names, which may contain sensitive information. + /// + /// + /// The default value can be overridden by setting the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + /// environment variable to "true". Explicitly setting this property will override the environment variable. + /// + /// + public bool EnableSensitiveData { get; set; } = TelemetryHelpers.EnableSensitiveDataDefault; + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType == typeof(ActivitySource) ? _activitySource : + base.GetService(serviceType, serviceKey); + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _activitySource.Dispose(); + _meter.Dispose(); + } + + base.Dispose(disposing); + } + + /// + public override async Task UploadAsync( + Stream content, string? mediaType = null, string? fileName = null, HostedFileUploadOptions? options = null, CancellationToken cancellationToken = default) + { + fileName ??= content is FileStream fs ? Path.GetFileName(fs.Name) : null; + mediaType ??= fileName is not null ? MediaTypeMap.GetMediaType(fileName) : null; + + using Activity? activity = StartActivity(UploadOperationName); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + if (activity is { IsAllDataRequested: true }) + { + if (mediaType is not null) + { + _ = activity.AddTag(FilesMediaTypeAttribute, mediaType); + } + + if (options?.Purpose is string purpose) + { + _ = activity.AddTag(FilesPurposeAttribute, purpose); + } + + if (options?.Scope is string scope) + { + _ = activity.AddTag(FilesScopeAttribute, scope); + } + + if (EnableSensitiveData && fileName is not null) + { + _ = activity.AddTag(OpenTelemetryConsts.File.Name, fileName); + } + + TagAdditionalProperties(activity, options); + } + + HostedFile? result = null; + Exception? error = null; + try + { + result = await base.UploadAsync(content, mediaType, fileName, options, cancellationToken).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + if (result is not null && activity is { IsAllDataRequested: true }) + { + _ = activity.AddTag(FilesIdAttribute, result.Id); + + if (result.SizeInBytes is long size) + { + _ = activity.AddTag(OpenTelemetryConsts.File.Size, size); + } + } + + RecordDuration(stopwatch, UploadOperationName, error); + SetErrorStatus(activity, error); + } + } + + /// + public override async Task DownloadAsync( + string fileId, HostedFileDownloadOptions? options = null, CancellationToken cancellationToken = default) + { + using Activity? activity = StartActivity(DownloadOperationName); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + if (activity is { IsAllDataRequested: true }) + { + _ = activity.AddTag(FilesIdAttribute, fileId); + + if (options?.Scope is string scope) + { + _ = activity.AddTag(FilesScopeAttribute, scope); + } + + TagAdditionalProperties(activity, options); + } + + Exception? error = null; + try + { + var result = await base.DownloadAsync(fileId, options, cancellationToken).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + RecordDuration(stopwatch, DownloadOperationName, error); + SetErrorStatus(activity, error); + } + } + + /// + public override async Task GetFileInfoAsync( + string fileId, HostedFileGetOptions? options = null, CancellationToken cancellationToken = default) + { + using Activity? activity = StartActivity(GetInfoOperationName); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + if (activity is { IsAllDataRequested: true }) + { + _ = activity.AddTag(FilesIdAttribute, fileId); + + if (options?.Scope is string scope) + { + _ = activity.AddTag(FilesScopeAttribute, scope); + } + + TagAdditionalProperties(activity, options); + } + + HostedFile? result = null; + Exception? error = null; + try + { + result = await base.GetFileInfoAsync(fileId, options, cancellationToken).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + if (result is not null && activity is { IsAllDataRequested: true }) + { + if (EnableSensitiveData && result.Name is string name) + { + _ = activity.AddTag(OpenTelemetryConsts.File.Name, name); + } + + if (result.SizeInBytes is long size) + { + _ = activity.AddTag(OpenTelemetryConsts.File.Size, size); + } + } + + RecordDuration(stopwatch, GetInfoOperationName, error); + SetErrorStatus(activity, error); + } + } + + /// + public override async IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using Activity? activity = StartActivity(ListOperationName); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + if (activity is { IsAllDataRequested: true }) + { + if (options?.Scope is string scope) + { + _ = activity.AddTag(FilesScopeAttribute, scope); + } + + if (options?.Purpose is string purpose) + { + _ = activity.AddTag(FilesPurposeAttribute, purpose); + } + + TagAdditionalProperties(activity, options); + } + + IAsyncEnumerator e; + Exception? error = null; + try + { + e = base.ListFilesAsync(options, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (Exception ex) + { + error = ex; + RecordDuration(stopwatch, ListOperationName, error); + SetErrorStatus(activity, error); + throw; + } + + int count = 0; + try + { + while (true) + { + try + { + if (!await e.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + error = ex; + throw; + } + + count++; + yield return e.Current; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + finally + { + if (activity is { IsAllDataRequested: true }) + { + _ = activity.AddTag(FilesListCountAttribute, count); + } + + RecordDuration(stopwatch, ListOperationName, error); + SetErrorStatus(activity, error); + + await e.DisposeAsync().ConfigureAwait(false); + } + } + + /// + public override async Task DeleteAsync( + string fileId, HostedFileDeleteOptions? options = null, CancellationToken cancellationToken = default) + { + using Activity? activity = StartActivity(DeleteOperationName); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + if (activity is { IsAllDataRequested: true }) + { + _ = activity.AddTag(FilesIdAttribute, fileId); + + if (options?.Scope is string scope) + { + _ = activity.AddTag(FilesScopeAttribute, scope); + } + + TagAdditionalProperties(activity, options); + } + + Exception? error = null; + try + { + return await base.DeleteAsync(fileId, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + RecordDuration(stopwatch, DeleteOperationName, error); + SetErrorStatus(activity, error); + } + } + + private static void SetErrorStatus(Activity? activity, Exception? error) + { + if (error is not null) + { + _ = activity? + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, error.Message); + } + } + + private void TagAdditionalProperties(Activity activity, HostedFileClientOptions? options) + { + if (EnableSensitiveData && options?.AdditionalProperties is { } props) + { + foreach (var prop in props) + { + _ = activity.AddTag(prop.Key, prop.Value); + } + } + } + + private Activity? StartActivity(string operationName) + { + Activity? activity = null; + if (_activitySource.HasListeners()) + { + activity = _activitySource.StartActivity( + operationName, + ActivityKind.Client); + + if (activity is { IsAllDataRequested: true }) + { + _ = activity + .AddTag(FilesOperationNameAttribute, operationName) + .AddTag(FilesProviderNameAttribute, _providerName); + + if (_serverAddress is not null) + { + _ = activity + .AddTag(OpenTelemetryConsts.Server.Address, _serverAddress) + .AddTag(OpenTelemetryConsts.Server.Port, _serverPort); + } + } + } + + return activity; + } + + private void RecordDuration(Stopwatch? stopwatch, string operationName, Exception? error) + { + if (_operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = default; + tags.Add(FilesOperationNameAttribute, operationName); + tags.Add(FilesProviderNameAttribute, _providerName); + + if (_serverAddress is string address) + { + tags.Add(OpenTelemetryConsts.Server.Address, address); + tags.Add(OpenTelemetryConsts.Server.Port, _serverPort); + } + + if (error is not null) + { + tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); + } + + _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClientBuilderExtensions.cs new file mode 100644 index 00000000000..8c5c0eba2f7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Files/OpenTelemetryHostedFileClientBuilderExtensions.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.AIFiles, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OpenTelemetryHostedFileClientBuilderExtensions +{ + /// + /// Adds OpenTelemetry support to the hosted file client pipeline. + /// + /// + /// Since there is currently no OpenTelemetry Semantic Convention for hosted file operations, this implementation + /// uses general client span conventions alongside standard file.* registry attributes where applicable. + /// The telemetry output is subject to change as relevant conventions emerge. + /// + /// The . + /// An optional to use to create a logger for logging events. + /// An optional source name that will be used on the telemetry data. + /// An optional callback that can be used to configure the instance. + /// The . + public static HostedFileClientBuilder UseOpenTelemetry( + this HostedFileClientBuilder builder, + ILoggerFactory? loggerFactory = null, + string? sourceName = null, + Action? configure = null) => + Throw.IfNull(builder).Use((innerClient, services) => + { + loggerFactory ??= services.GetService(); + + var client = new OpenTelemetryHostedFileClient(innerClient, loggerFactory?.CreateLogger(typeof(OpenTelemetryHostedFileClient)), sourceName); + configure?.Invoke(client); + + return client; + }); +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj index c0d709498e2..b255c9a0474 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj @@ -32,6 +32,7 @@ true + true true true true diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 04a33a75be5..0f13f5ac1de 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -156,6 +156,12 @@ public static class Usage } } + public static class File + { + public const string Name = "file.name"; + public const string Size = "file.size"; + } + public static class Server { public const string Address = "server.address"; diff --git a/src/Shared/DiagnosticIds/DiagnosticIds.cs b/src/Shared/DiagnosticIds/DiagnosticIds.cs index 0b11c260d10..c3d4f6581c7 100644 --- a/src/Shared/DiagnosticIds/DiagnosticIds.cs +++ b/src/Shared/DiagnosticIds/DiagnosticIds.cs @@ -57,6 +57,7 @@ internal static class Experiments internal const string AIResponseContinuations = AIExperiments; internal const string AICodeInterpreter = AIExperiments; internal const string AIRealTime = AIExperiments; + internal const string AIFiles = AIExperiments; // These diagnostic IDs are defined by the OpenAI package for its experimental APIs. // We use the same IDs so consumers do not need to suppress additional diagnostics diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/DelegatingHostedFileClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/DelegatingHostedFileClientTests.cs new file mode 100644 index 00000000000..254ec72c14e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/DelegatingHostedFileClientTests.cs @@ -0,0 +1,256 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class DelegatingHostedFileClientTests +{ + [Fact] + public void RequiresInnerClient() + { + Assert.Throws("innerClient", () => new NoOpDelegatingHostedFileClient(null!)); + } + + [Fact] + public async Task UploadAsyncDefaultsToInnerClientAsync() + { + var expectedStream = new MemoryStream([1, 2, 3]); + var expectedMediaType = "text/plain"; + var expectedFileName = "test.txt"; + var expectedOptions = new HostedFileUploadOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + var expectedFile = new HostedFile("file-123"); + using var inner = new TestHostedFileClient + { + UploadAsyncCallback = (content, mediaType, fileName, options, cancellationToken) => + { + Assert.Same(expectedStream, content); + Assert.Equal(expectedMediaType, mediaType); + Assert.Equal(expectedFileName, fileName); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var resultTask = delegating.UploadAsync(expectedStream, expectedMediaType, expectedFileName, expectedOptions, expectedCancellationToken); + Assert.False(resultTask.IsCompleted); + expectedResult.SetResult(expectedFile); + Assert.True(resultTask.IsCompleted); + Assert.Same(expectedFile, await resultTask); + } + + [Fact] + public async Task DownloadAsyncDefaultsToInnerClientAsync() + { + var expectedFileId = "file-456"; + var expectedOptions = new HostedFileDownloadOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + using var inner = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, cancellationToken) => + { + Assert.Equal(expectedFileId, fileId); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var resultTask = delegating.DownloadAsync(expectedFileId, expectedOptions, expectedCancellationToken); + Assert.False(resultTask.IsCompleted); + using var downloadStream = new TestHostedFileDownloadStream([1, 2, 3]); + expectedResult.SetResult(downloadStream); + Assert.True(resultTask.IsCompleted); + Assert.Same(downloadStream, await resultTask); + } + + [Fact] + public async Task GetFileInfoAsyncDefaultsToInnerClientAsync() + { + var expectedFileId = "file-789"; + var expectedOptions = new HostedFileGetOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + var expectedFile = new HostedFile("file-789") { Name = "info.txt" }; + using var inner = new TestHostedFileClient + { + GetFileInfoAsyncCallback = (fileId, options, cancellationToken) => + { + Assert.Equal(expectedFileId, fileId); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var resultTask = delegating.GetFileInfoAsync(expectedFileId, expectedOptions, expectedCancellationToken); + Assert.False(resultTask.IsCompleted); + expectedResult.SetResult(expectedFile); + Assert.True(resultTask.IsCompleted); + Assert.Same(expectedFile, await resultTask); + } + + [Fact] + public async Task ListFilesAsyncDefaultsToInnerClientAsync() + { + var expectedOptions = new HostedFileListOptions(); + var expectedCancellationToken = CancellationToken.None; + HostedFile[] expectedFiles = + [ + new HostedFile("file-1"), + new HostedFile("file-2") + ]; + + using var inner = new TestHostedFileClient + { + ListFilesAsyncCallback = (options, cancellationToken) => + { + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return YieldAsync(expectedFiles); + } + }; + + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var resultAsyncEnumerable = delegating.ListFilesAsync(expectedOptions, expectedCancellationToken); + var enumerator = resultAsyncEnumerable.GetAsyncEnumerator(); + Assert.True(await enumerator.MoveNextAsync()); + Assert.Same(expectedFiles[0], enumerator.Current); + Assert.True(await enumerator.MoveNextAsync()); + Assert.Same(expectedFiles[1], enumerator.Current); + Assert.False(await enumerator.MoveNextAsync()); + } + + [Fact] + public async Task DeleteAsyncDefaultsToInnerClientAsync() + { + var expectedFileId = "file-del"; + var expectedOptions = new HostedFileDeleteOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + using var inner = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, cancellationToken) => + { + Assert.Equal(expectedFileId, fileId); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var resultTask = delegating.DeleteAsync(expectedFileId, expectedOptions, expectedCancellationToken); + Assert.False(resultTask.IsCompleted); + expectedResult.SetResult(true); + Assert.True(resultTask.IsCompleted); + Assert.True(await resultTask); + } + + [Fact] + public void GetServiceThrowsForNullType() + { + using var inner = new TestHostedFileClient(); + using var delegating = new NoOpDelegatingHostedFileClient(inner); + Assert.Throws("serviceType", () => delegating.GetService(null!)); + } + + [Fact] + public void GetServiceReturnsSelfForIHostedFileClientType() + { + using var inner = new TestHostedFileClient(); + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var client = delegating.GetService(typeof(IHostedFileClient)); + Assert.Same(delegating, client); + } + + [Fact] + public void GetServiceDelegatesToInnerForOtherTypes() + { + var expectedResult = TimeZoneInfo.Local; + var expectedKey = new object(); + using var inner = new TestHostedFileClient + { + GetServiceCallback = (type, key) => type == typeof(TimeZoneInfo) && key == expectedKey + ? expectedResult + : throw new InvalidOperationException("Unexpected call") + }; + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var tzi = delegating.GetService(typeof(TimeZoneInfo), expectedKey); + Assert.Same(expectedResult, tzi); + } + + [Fact] + public void GetServiceDelegatesToInnerIfKeyIsNotNull() + { + var expectedKey = new object(); + var expectedResult = "some-service"; + using var inner = new TestHostedFileClient + { + GetServiceCallback = (type, key) => type == typeof(string) && key == expectedKey + ? expectedResult + : throw new InvalidOperationException("Unexpected call") + }; + using var delegating = new NoOpDelegatingHostedFileClient(inner); + var result = delegating.GetService(typeof(string), expectedKey); + Assert.Same(expectedResult, result); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable input) + { + await Task.Yield(); + foreach (var item in input) + { + yield return item; + } + } + + private sealed class NoOpDelegatingHostedFileClient(IHostedFileClient innerClient) + : DelegatingHostedFileClient(innerClient); + + private sealed class TestHostedFileDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public TestHostedFileDownloadStream(byte[] data) + { + _inner = new MemoryStream(data); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientExtensionsTests.cs new file mode 100644 index 00000000000..77beac7a2ab --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientExtensionsTests.cs @@ -0,0 +1,505 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class HostedFileClientExtensionsTests +{ + [Fact] + public async Task UploadAsync_DataContent_PassesCorrectArgs() + { + var data = new byte[] { 10, 20, 30 }; + var content = new DataContent(data, "application/pdf") { Name = "doc.pdf" }; + Stream? capturedStream = null; + string? capturedMediaType = null; + string? capturedName = null; + + using var client = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + { + capturedStream = stream; + capturedMediaType = mediaType; + capturedName = fileName; + return Task.FromResult(new HostedFile("file-1")); + } + }; + var result = await client.UploadAsync(content); + Assert.NotNull(capturedStream); + capturedStream!.Position = 0; + var buffer = new byte[capturedStream.Length]; + _ = await capturedStream.ReadAsync(buffer, 0, buffer.Length); + Assert.Equal(data, buffer); + Assert.Equal("application/pdf", capturedMediaType); + Assert.Equal("doc.pdf", capturedName); + Assert.Equal("file-1", result.Id); + } + + [Fact] + public async Task UploadAsync_DataContent_NullClient_Throws() + { + IHostedFileClient client = null!; + var content = new DataContent(new byte[] { 1 }, "text/plain"); + await Assert.ThrowsAsync("client", () => client.UploadAsync(content)); + } + + [Fact] + public async Task UploadAsync_DataContent_NullContent_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("content", () => client.UploadAsync((DataContent)null!)); + } + + [Fact] + public async Task UploadAsync_FilePath_CreatesStreamAndInfersMediaType() + { + var tempFile = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid()}.txt"); + var fileContent = new byte[] { 65, 66, 67 }; // ABC + await File.WriteAllBytesAsync(tempFile, fileContent); + + try + { + Stream? capturedStream = null; + string? capturedMediaType = null; + string? capturedFileName = null; + + using var client = new TestHostedFileClient + { + UploadAsyncCallback = async (stream, mediaType, fileName, options, ct) => + { + var ms = new MemoryStream(); + await stream.CopyToAsync(ms, 81920, ct); + capturedStream = ms; + capturedMediaType = mediaType; + capturedFileName = fileName; + return new HostedFile("file-2"); + } + }; + var result = await client.UploadAsync(tempFile); + Assert.NotNull(capturedStream); + Assert.Equal(fileContent, ((MemoryStream)capturedStream!).ToArray()); + Assert.Equal("text/plain", capturedMediaType); + Assert.Equal(Path.GetFileName(tempFile), capturedFileName); + Assert.Equal("file-2", result.Id); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public async Task UploadAsync_FilePath_NullClient_Throws() + { + IHostedFileClient client = null!; + await Assert.ThrowsAsync("client", () => client.UploadAsync("somefile.txt")); + } + + [Fact] + public async Task UploadAsync_FilePath_NullPath_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("filePath", () => client.UploadAsync((string)null!)); + } + + [Fact] + public async Task UploadAsync_FilePath_EmptyPath_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("filePath", () => client.UploadAsync(string.Empty)); + } + + [Fact] + public async Task DownloadToAsync_SavesStreamToFilePath() + { + var data = new byte[] { 1, 2, 3, 4, 5 }; + var tempFile = Path.Combine(Path.GetTempPath(), $"download-{Guid.NewGuid()}.bin"); + + using var client = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + { + Assert.Equal("file-dl", fileId); + return Task.FromResult(new TestHostedFileDownloadStream(data)); + } + }; + + try + { + var savedPath = await client.DownloadToAsync("file-dl", tempFile); + Assert.Equal(tempFile, savedPath); + Assert.Equal(data, await File.ReadAllBytesAsync(tempFile)); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public async Task DownloadToAsync_DirectoryPath_UsesFileName() + { + var data = new byte[] { 10, 20 }; + var tempDir = Path.Combine(Path.GetTempPath(), $"dldir-{Guid.NewGuid()}"); + Directory.CreateDirectory(tempDir); + + using var client = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + Task.FromResult(new TestHostedFileDownloadStream(data, fileName: "result.bin")) + }; + + try + { + var savedPath = await client.DownloadToAsync("file-dl2", tempDir); + Assert.Equal(Path.Combine(tempDir, "result.bin"), savedPath); + Assert.Equal(data, await File.ReadAllBytesAsync(savedPath)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task DownloadToAsync_DirectoryPath_NoFileName_UsesFileId() + { + var data = new byte[] { 99 }; + var tempDir = Path.Combine(Path.GetTempPath(), $"dldir2-{Guid.NewGuid()}"); + Directory.CreateDirectory(tempDir); + + using var client = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + Task.FromResult(new TestHostedFileDownloadStream(data)) + }; + + try + { + var savedPath = await client.DownloadToAsync("file-id-fallback", tempDir); + Assert.Equal(Path.Combine(tempDir, "file-id-fallback"), savedPath); + Assert.Equal(data, await File.ReadAllBytesAsync(savedPath)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task DownloadToAsync_NullClient_Throws() + { + IHostedFileClient client = null!; + await Assert.ThrowsAsync("client", () => client.DownloadToAsync("file-1", "path")); + } + + [Fact] + public async Task DownloadToAsync_NullFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadToAsync(null!, "path")); + } + + [Fact] + public async Task DownloadToAsync_NullDestinationPath_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("destinationPath", () => client.DownloadToAsync("file-1", null!)); + } + + [Fact] + public async Task DownloadToAsync_EmptyFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadToAsync(string.Empty, "path")); + } + + [Fact] + public async Task DownloadToAsync_WhitespaceFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadToAsync(" ", "path")); + } + + [Fact] + public async Task DownloadAsync_HostedFileContent_PassesFileId() + { + var hostedFileContent = new HostedFileContent("file-hfc"); + string? capturedFileId = null; + + using var client = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + { + capturedFileId = fileId; + return Task.FromResult(new TestHostedFileDownloadStream([])); + } + }; + using var stream = await client.DownloadAsync(hostedFileContent); + Assert.Equal("file-hfc", capturedFileId); + } + + [Fact] + public async Task DownloadAsync_HostedFileContent_NullClient_Throws() + { + IHostedFileClient client = null!; + var content = new HostedFileContent("file-1"); + await Assert.ThrowsAsync("client", () => client.DownloadAsync(content)); + } + + [Fact] + public async Task DownloadAsync_HostedFileContent_NullContent_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("hostedFile", () => client.DownloadAsync((HostedFileContent)null!)); + } + + [Fact] + public async Task DownloadAsDataContentAsync_ReturnsCorrectDataContent() + { + var data = new byte[] { 7, 8, 9 }; + + using var client = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + { + Assert.Equal("file-dc", fileId); + return Task.FromResult( + new TestHostedFileDownloadStream(data, mediaType: "image/png", fileName: "photo.png")); + } + }; + var result = await client.DownloadAsDataContentAsync("file-dc"); + Assert.Equal(data, result.Data.ToArray()); + Assert.Equal("image/png", result.MediaType); + Assert.Equal("photo.png", result.Name); + } + + [Fact] + public async Task DownloadAsDataContentAsync_NullClient_Throws() + { + IHostedFileClient client = null!; + await Assert.ThrowsAsync("client", () => client.DownloadAsDataContentAsync("file-1")); + } + + [Fact] + public async Task DownloadAsDataContentAsync_NullFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadAsDataContentAsync(null!)); + } + + [Fact] + public async Task DownloadAsDataContentAsync_EmptyFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadAsDataContentAsync(string.Empty)); + } + + [Fact] + public async Task DownloadAsDataContentAsync_WhitespaceFileId_Throws() + { + using var client = new TestHostedFileClient(); + await Assert.ThrowsAsync("fileId", () => client.DownloadAsDataContentAsync(" ")); + } + + [Fact] + public void GetMetadata_CallsGetServiceWithCorrectType() + { + var expectedMetadata = new HostedFileClientMetadata("test-provider"); + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(HostedFileClientMetadata), type); + Assert.Null(key); + return expectedMetadata; + } + }; + var result = client.GetMetadata(); + Assert.Same(expectedMetadata, result); + } + + [Fact] + public void GetMetadata_NullClient_Throws() + { + IHostedFileClient client = null!; + Assert.Throws("client", () => client.GetMetadata()); + } + + [Fact] + public void GetServiceGeneric_CallsGetServiceWithCorrectType() + { + var expectedResult = "some-service-value"; + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(string), type); + Assert.Null(key); + return expectedResult; + } + }; + var result = client.GetService(); + Assert.Same(expectedResult, result); + } + + [Fact] + public void GetServiceGeneric_WithKey_PassesKey() + { + var expectedKey = new object(); + var expectedResult = 42; + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(int), type); + Assert.Same(expectedKey, key); + return expectedResult; + } + }; + var result = client.GetService(expectedKey); + Assert.Equal(42, result); + } + + [Fact] + public void GetServiceGeneric_NullClient_Throws() + { + IHostedFileClient client = null!; + Assert.Throws("client", () => client.GetService()); + } + + [Fact] + public void GetRequiredService_ReturnsService() + { + var expectedResult = "some-service-value"; + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(string), type); + Assert.Null(key); + return expectedResult; + } + }; + var result = client.GetRequiredService(); + Assert.Same(expectedResult, result); + } + + [Fact] + public void GetRequiredService_WithKey_PassesKey() + { + var expectedKey = new object(); + var expectedResult = 42; + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(int), type); + Assert.Same(expectedKey, key); + return expectedResult; + } + }; + var result = client.GetRequiredService(expectedKey); + Assert.Equal(42, result); + } + + [Fact] + public void GetRequiredService_NotFound_Throws() + { + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => null + }; + Assert.Throws(() => client.GetRequiredService()); + } + + [Fact] + public void GetRequiredService_NullClient_Throws() + { + IHostedFileClient client = null!; + Assert.Throws("client", () => client.GetRequiredService()); + } + + [Fact] + public void GetRequiredServiceNonGeneric_ReturnsService() + { + var expectedResult = "some-service-value"; + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => + { + Assert.Equal(typeof(string), type); + Assert.Null(key); + return expectedResult; + } + }; + var result = client.GetRequiredService(typeof(string)); + Assert.Same(expectedResult, result); + } + + [Fact] + public void GetRequiredServiceNonGeneric_NotFound_Throws() + { + using var client = new TestHostedFileClient + { + GetServiceCallback = (type, key) => null + }; + Assert.Throws(() => client.GetRequiredService(typeof(string))); + } + + [Fact] + public void GetRequiredServiceNonGeneric_NullClient_Throws() + { + IHostedFileClient client = null!; + Assert.Throws("client", () => client.GetRequiredService(typeof(string))); + } + + [Fact] + public void GetRequiredServiceNonGeneric_NullServiceType_Throws() + { + using var client = new TestHostedFileClient(); + Assert.Throws("serviceType", () => client.GetRequiredService(null!)); + } + + private sealed class TestHostedFileDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public TestHostedFileDownloadStream(byte[] data, string? mediaType = null, string? fileName = null) + { + _inner = new MemoryStream(data); + MediaType = mediaType; + FileName = fileName; + } + + public override string? MediaType { get; } + public override string? FileName { get; } + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientMetadataTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientMetadataTests.cs new file mode 100644 index 00000000000..54d416b455c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientMetadataTests.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class HostedFileClientMetadataTests +{ + [Fact] + public void DefaultConstructor_PropertiesAreNull() + { + var metadata = new HostedFileClientMetadata(); + Assert.Null(metadata.ProviderName); + Assert.Null(metadata.ProviderUri); + } + + [Fact] + public void Constructor_WithBothValues_Roundtrips() + { + var uri = new Uri("https://api.openai.com"); + var metadata = new HostedFileClientMetadata("openai", uri); + Assert.Equal("openai", metadata.ProviderName); + Assert.Same(uri, metadata.ProviderUri); + } + + [Fact] + public void Constructor_WithOnlyProviderName() + { + var metadata = new HostedFileClientMetadata(providerName: "anthropic"); + Assert.Equal("anthropic", metadata.ProviderName); + Assert.Null(metadata.ProviderUri); + } + + [Fact] + public void Constructor_WithOnlyProviderUri() + { + var uri = new Uri("https://api.example.com"); + var metadata = new HostedFileClientMetadata(providerUri: uri); + Assert.Null(metadata.ProviderName); + Assert.Same(uri, metadata.ProviderUri); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientOptionsTests.cs new file mode 100644 index 00000000000..07932951e7a --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileClientOptionsTests.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class HostedFileClientOptionsTests +{ + [Fact] + public void UploadOptions_PropsDefault() + { + var options = new HostedFileUploadOptions(); + Assert.Null(options.Purpose); + Assert.Null(options.Scope); + Assert.Null(options.AdditionalProperties); + } + + [Fact] + public void UploadOptions_PropsRoundtrip() + { + var props = new AdditionalPropertiesDictionary { { "key", "value" } }; + var options = new HostedFileUploadOptions + { + Purpose = "fine-tune", + Scope = "container-1", + AdditionalProperties = props + }; + + Assert.Equal("fine-tune", options.Purpose); + Assert.Equal("container-1", options.Scope); + Assert.Same(props, options.AdditionalProperties); + } + + [Fact] + public void DownloadOptions_PropsDefault() + { + var options = new HostedFileDownloadOptions(); + Assert.Null(options.Scope); + Assert.Null(options.AdditionalProperties); + } + + [Fact] + public void DownloadOptions_PropsRoundtrip() + { + var props = new AdditionalPropertiesDictionary { { "key", "value" } }; + var options = new HostedFileDownloadOptions + { + Scope = "scope-1", + AdditionalProperties = props + }; + + Assert.Equal("scope-1", options.Scope); + Assert.Same(props, options.AdditionalProperties); + } + + [Fact] + public void GetOptions_PropsDefault() + { + var options = new HostedFileGetOptions(); + Assert.Null(options.Scope); + Assert.Null(options.AdditionalProperties); + } + + [Fact] + public void GetOptions_PropsRoundtrip() + { + var props = new AdditionalPropertiesDictionary { { "k", "v" } }; + var options = new HostedFileGetOptions + { + Scope = "scope-2", + AdditionalProperties = props + }; + + Assert.Equal("scope-2", options.Scope); + Assert.Same(props, options.AdditionalProperties); + } + + [Fact] + public void DeleteOptions_PropsDefault() + { + var options = new HostedFileDeleteOptions(); + Assert.Null(options.Scope); + Assert.Null(options.AdditionalProperties); + } + + [Fact] + public void DeleteOptions_PropsRoundtrip() + { + var props = new AdditionalPropertiesDictionary { { "k", "v" } }; + var options = new HostedFileDeleteOptions + { + Scope = "scope-3", + AdditionalProperties = props + }; + + Assert.Equal("scope-3", options.Scope); + Assert.Same(props, options.AdditionalProperties); + } + + [Fact] + public void ListOptions_PropsDefault() + { + var options = new HostedFileListOptions(); + Assert.Null(options.Purpose); + Assert.Null(options.Limit); + Assert.Null(options.Scope); + Assert.Null(options.AdditionalProperties); + } + + [Fact] + public void ListOptions_PropsRoundtrip() + { + var props = new AdditionalPropertiesDictionary { { "k", "v" } }; + var options = new HostedFileListOptions + { + Purpose = "assistants", + Limit = 50, + Scope = "scope-4", + AdditionalProperties = props + }; + + Assert.Equal("assistants", options.Purpose); + Assert.Equal(50, options.Limit); + Assert.Equal("scope-4", options.Scope); + Assert.Same(props, options.AdditionalProperties); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileDownloadStreamTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileDownloadStreamTests.cs new file mode 100644 index 00000000000..fad1a834374 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileDownloadStreamTests.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class HostedFileDownloadStreamTests +{ + [Fact] + public void Defaults_ReturnNull() + { + using var stream = new MinimalDownloadStream([1, 2, 3]); + Assert.Null(stream.MediaType); + Assert.Null(stream.FileName); + } + + [Fact] + public async Task ToDataContentAsync_BuffersStreamContent() + { + var data = new byte[] { 10, 20, 30, 40, 50 }; + using var stream = new MetadataDownloadStream(data, "application/json", "data.json"); + var content = await stream.ToDataContentAsync(); + Assert.Equal(data, content.Data.ToArray()); + Assert.Equal("application/json", content.MediaType); + Assert.Equal("data.json", content.Name); + } + + [Fact] + public async Task ToDataContentAsync_NullMediaType_DefaultsToOctetStream() + { + var data = new byte[] { 1, 2 }; + using var stream = new MinimalDownloadStream(data); + var content = await stream.ToDataContentAsync(); + Assert.Equal(data, content.Data.ToArray()); + Assert.Equal("application/octet-stream", content.MediaType); + Assert.Null(content.Name); + } + + [Fact] + public async Task ToDataContentAsync_EmptyStream_ReturnsEmptyData() + { + using var stream = new MinimalDownloadStream([]); + var content = await stream.ToDataContentAsync(); + Assert.Empty(content.Data.ToArray()); + } + + /// + /// Minimal implementation that does not override MediaType or FileName, testing the default behavior. + /// + private sealed class MinimalDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public MinimalDownloadStream(byte[] data) + { + _inner = new MemoryStream(data); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } + + /// + /// Implementation that provides MediaType and FileName metadata. + /// + private sealed class MetadataDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public MetadataDownloadStream(byte[] data, string? mediaType, string? fileName) + { + _inner = new MemoryStream(data); + MediaType = mediaType; + FileName = fileName; + } + + public override string? MediaType { get; } + public override string? FileName { get; } + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileTests.cs new file mode 100644 index 00000000000..d6f3e80e9ae --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Files/HostedFileTests.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class HostedFileTests +{ + [Fact] + public void Constructor_NullId_Throws() + { + Assert.Throws("id", () => new HostedFile(null!)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("\t")] + public void Constructor_EmptyOrWhitespaceId_Throws(string id) + { + Assert.Throws(nameof(id), () => new HostedFile(id)); + } + + [Fact] + public void Constructor_PropsDefault() + { + var file = new HostedFile("file-123"); + Assert.Equal("file-123", file.Id); + Assert.Null(file.Name); + Assert.Null(file.MediaType); + Assert.Null(file.SizeInBytes); + Assert.Null(file.CreatedAt); + Assert.Null(file.Purpose); + Assert.Null(file.AdditionalProperties); + Assert.Null(file.RawRepresentation); + } + + [Fact] + public void Constructor_PropsRoundtrip() + { + var file = new HostedFile("file-123"); + var now = DateTimeOffset.UtcNow; + var props = new AdditionalPropertiesDictionary { { "key", "value" } }; + var raw = new object(); + + file.Name = "test.txt"; + file.MediaType = "text/plain"; + file.SizeInBytes = 12345L; + file.CreatedAt = now; + file.Purpose = "assistants"; + file.AdditionalProperties = props; + file.RawRepresentation = raw; + + Assert.Equal("test.txt", file.Name); + Assert.Equal("text/plain", file.MediaType); + Assert.Equal(12345L, file.SizeInBytes); + Assert.Equal(now, file.CreatedAt); + Assert.Equal("assistants", file.Purpose); + Assert.Same(props, file.AdditionalProperties); + Assert.Same(raw, file.RawRepresentation); + } + + [Fact] + public void ToHostedFileContent_CreatesCorrectContent() + { + var file = new HostedFile("file-123") + { + Name = "test.txt", + MediaType = "text/plain" + }; + + HostedFileContent content = file.ToHostedFileContent(); + + Assert.Equal("file-123", content.FileId); + Assert.Equal("test.txt", content.Name); + Assert.Equal("text/plain", content.MediaType); + } + + [Fact] + public void ToHostedFileContent_NullOptionalProperties() + { + var file = new HostedFile("file-456"); + + HostedFileContent content = file.ToHostedFileContent(); + + Assert.Equal("file-456", content.FileId); + Assert.Null(content.Name); + Assert.Null(content.MediaType); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestHostedFileClient.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestHostedFileClient.cs new file mode 100644 index 00000000000..e5ebfb256be --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestHostedFileClient.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +internal sealed class TestHostedFileClient : IHostedFileClient +{ + public TestHostedFileClient() + { + GetServiceCallback = DefaultGetServiceCallback; + } + + public Func>? UploadAsyncCallback { get; set; } + + public Func>? DownloadAsyncCallback { get; set; } + + public Func>? GetFileInfoAsyncCallback { get; set; } + + public Func>? ListFilesAsyncCallback { get; set; } + + public Func>? DeleteAsyncCallback { get; set; } + + public Func GetServiceCallback { get; set; } + + private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) => + serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public Task UploadAsync( + Stream content, + string? mediaType = null, + string? fileName = null, + HostedFileUploadOptions? options = null, + CancellationToken cancellationToken = default) => + UploadAsyncCallback!.Invoke(content, mediaType, fileName, options, cancellationToken); + + public Task DownloadAsync( + string fileId, + HostedFileDownloadOptions? options = null, + CancellationToken cancellationToken = default) => + DownloadAsyncCallback!.Invoke(fileId, options, cancellationToken); + + public Task GetFileInfoAsync( + string fileId, + HostedFileGetOptions? options = null, + CancellationToken cancellationToken = default) => + GetFileInfoAsyncCallback!.Invoke(fileId, options, cancellationToken); + + public IAsyncEnumerable ListFilesAsync( + HostedFileListOptions? options = null, + CancellationToken cancellationToken = default) => + ListFilesAsyncCallback!.Invoke(options, cancellationToken); + + public Task DeleteAsync( + string fileId, + HostedFileDeleteOptions? options = null, + CancellationToken cancellationToken = default) => + DeleteAsyncCallback!.Invoke(fileId, options, cancellationToken); + + public object? GetService(Type serviceType, object? serviceKey = null) => + GetServiceCallback(serviceType, serviceKey); + + void IDisposable.Dispose() + { + // No resources need disposing. + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientIntegrationTests.cs new file mode 100644 index 00000000000..dfe47c1aea5 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientIntegrationTests.cs @@ -0,0 +1,291 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Microsoft.TestUtilities; +using Xunit; + +#pragma warning disable MEAI001 + +namespace Microsoft.Extensions.AI; + +public sealed class OpenAIHostedFileClientIntegrationTests : IDisposable +{ + private readonly IHostedFileClient? _client = IntegrationTestHelpers.GetOpenAIClient()?.AsIHostedFileClient(); + + public void Dispose() + { + _client?.Dispose(); + } + + /// Retries a download operation to handle the delay between upload and file availability. + private static async Task RetryAsync(Func> action, int maxRetries = 5, int delayMs = 2000) + { + for (int i = 0; i < maxRetries; i++) + { + try + { + return await action(); + } + catch (ClientResultException) when (i < maxRetries - 1) + { + await Task.Delay(delayMs * (i + 1)); + } + } + + return await action(); + } + + private void SkipIfNotEnabled() + { + string? skipIntegration = TestRunnerConfiguration.Instance["SkipIntegrationTests"]; + + if (skipIntegration is not null || _client is null) + { + throw new SkipTestException("Client is not enabled."); + } + } + + [ConditionalFact] + public async Task Upload_Download_Delete_Roundtrip() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.jsonl"; + string content = """{"prompt": "hello", "completion": "world"}"""; + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + string? uploadedFileId = null; + + try + { + // Upload with "fine-tune" purpose since "assistants" files cannot be downloaded via the Files API. + using var uploadStream = new MemoryStream(contentBytes); + var uploadedFile = await _client!.UploadAsync(uploadStream, "application/jsonl", fileName, new HostedFileUploadOptions { Purpose = "fine-tune" }); + uploadedFileId = uploadedFile.Id; + + Assert.NotNull(uploadedFile.Id); + Assert.NotEmpty(uploadedFile.Id); + Assert.Equal(fileName, uploadedFile.Name); + Assert.Equal("application/x-ndjson", uploadedFile.MediaType); + Assert.NotNull(uploadedFile.Purpose); + Assert.NotNull(uploadedFile.SizeInBytes); + Assert.Equal(contentBytes.Length, uploadedFile.SizeInBytes); + Assert.NotNull(uploadedFile.CreatedAt); + Assert.NotNull(uploadedFile.RawRepresentation); + + // Download (with retry - files may not be immediately available) + string downloadedContent = await RetryAsync(async () => + { + using var downloadStream = await _client.DownloadAsync(uploadedFileId); + Assert.Equal("application/x-ndjson", downloadStream.MediaType); + Assert.Equal(fileName, downloadStream.FileName); + using var reader = new StreamReader(downloadStream, Encoding.UTF8); + return await reader.ReadToEndAsync(); + }); + + Assert.Equal(content, downloadedContent); + + // Delete + bool deleted = await _client.DeleteAsync(uploadedFileId); + Assert.True(deleted); + uploadedFileId = null; + + // Verify not found after deletion + var fileInfo = await _client.GetFileInfoAsync(uploadedFile.Id); + Assert.Null(fileInfo); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + } + } + + [ConditionalFact] + public async Task Upload_ListFiles_VerifyPresent() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.txt"; + string? uploadedFileId = null; + + try + { + using var uploadStream = new MemoryStream("list test"u8.ToArray()); + var uploadedFile = await _client!.UploadAsync(uploadStream, "text/plain", fileName, new HostedFileUploadOptions { Purpose = "assistants" }); + uploadedFileId = uploadedFile.Id; + + var files = new List(); + await foreach (var file in _client.ListFilesAsync(new HostedFileListOptions { Purpose = "assistants" })) + { + files.Add(file); + } + + Assert.Contains(files, f => f.Id == uploadedFileId && f.MediaType == "text/plain" && f.Name == fileName); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + } + } + + [ConditionalFact] + public async Task GetFileInfo_ReturnsMetadata() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.txt"; + byte[] contentBytes = "metadata test"u8.ToArray(); + string? uploadedFileId = null; + + try + { + using var uploadStream = new MemoryStream(contentBytes); + var uploadedFile = await _client!.UploadAsync(uploadStream, "text/plain", fileName, new HostedFileUploadOptions { Purpose = "assistants" }); + uploadedFileId = uploadedFile.Id; + + var fileInfo = await _client.GetFileInfoAsync(uploadedFileId); + + Assert.NotNull(fileInfo); + Assert.Equal(uploadedFileId, fileInfo.Id); + Assert.Equal(fileName, fileInfo.Name); + Assert.Equal("text/plain", fileInfo.MediaType); + Assert.NotNull(fileInfo.SizeInBytes); + Assert.True(fileInfo.SizeInBytes > 0); + Assert.NotNull(fileInfo.Purpose); + Assert.NotNull(fileInfo.CreatedAt); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + } + } + + [ConditionalFact] + public async Task Delete_NonExistent_ReturnsFalse() + { + SkipIfNotEnabled(); + + bool deleted = await _client!.DeleteAsync("file-nonexistent000000000000"); + Assert.False(deleted); + } + + [ConditionalFact] + public async Task GetFileInfo_NonExistent_ReturnsNull() + { + SkipIfNotEnabled(); + + var fileInfo = await _client!.GetFileInfoAsync("file-nonexistent000000000000"); + Assert.Null(fileInfo); + } + + [ConditionalFact] + public async Task Upload_DataContent_Extension() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.txt"; + byte[] contentBytes = "data content test"u8.ToArray(); + string? uploadedFileId = null; + + try + { + var dataContent = new DataContent(contentBytes, "text/plain") { Name = fileName }; + var uploadedFile = await _client!.UploadAsync(dataContent, new HostedFileUploadOptions { Purpose = "assistants" }); + uploadedFileId = uploadedFile.Id; + + Assert.NotNull(uploadedFile.Id); + Assert.NotEmpty(uploadedFile.Id); + Assert.Equal("text/plain", uploadedFile.MediaType); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + } + } + + [ConditionalFact] + public async Task Download_AsDataContent_Extension() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.jsonl"; + string content = """{"prompt": "hello", "completion": "world"}"""; + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + string? uploadedFileId = null; + + try + { + using var uploadStream = new MemoryStream(contentBytes); + var uploadedFile = await _client!.UploadAsync(uploadStream, "application/jsonl", fileName, new HostedFileUploadOptions { Purpose = "fine-tune" }); + uploadedFileId = uploadedFile.Id; + + var dataContent = await RetryAsync(() => _client.DownloadAsDataContentAsync(uploadedFileId)); + Assert.Equal(content, Encoding.UTF8.GetString(dataContent.Data.ToArray())); + Assert.Equal("application/x-ndjson", dataContent.MediaType); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + } + } + + [ConditionalFact] + public async Task Upload_DownloadTo_Extension() + { + SkipIfNotEnabled(); + + string fileName = $"test-{Guid.NewGuid():N}.jsonl"; + string content = """{"prompt": "hello", "completion": "world"}"""; + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + string? uploadedFileId = null; + string? tempDir = null; + + try + { + using var uploadStream = new MemoryStream(contentBytes); + var uploadedFile = await _client!.UploadAsync(uploadStream, "application/jsonl", fileName, new HostedFileUploadOptions { Purpose = "fine-tune" }); + uploadedFileId = uploadedFile.Id; + + tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + string savedPath = await RetryAsync(() => _client.DownloadToAsync(uploadedFileId, tempDir)); + + Assert.True(File.Exists(savedPath)); + string savedContent = File.ReadAllText(savedPath); + Assert.Equal(content, savedContent); + } + finally + { + if (uploadedFileId is not null) + { + await _client!.DeleteAsync(uploadedFileId); + } + + if (tempDir is not null && Directory.Exists(tempDir)) + { + Directory.Delete(tempDir, recursive: true); + } + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientTests.cs new file mode 100644 index 00000000000..de01e01c424 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIHostedFileClientTests.cs @@ -0,0 +1,891 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using OpenAI; +using OpenAI.Containers; +using OpenAI.Files; +using Xunit; + +#pragma warning disable S103 // Lines should not be too long +#pragma warning disable MEAI001 + +namespace Microsoft.Extensions.AI; + +public class OpenAIHostedFileClientTests +{ + [Fact] + public void AsIHostedFileClient_OpenAIClient_NullThrows() + { + Assert.Throws("openAIClient", () => ((OpenAIClient)null!).AsIHostedFileClient()); + } + + [Fact] + public void AsIHostedFileClient_OpenAIFileClient_NullThrows() + { + Assert.Throws("fileClient", () => ((OpenAIFileClient)null!).AsIHostedFileClient()); + } + + [Fact] + public void AsIHostedFileClient_ContainerClient_NullThrows() + { + Assert.Throws("containerClient", () => ((ContainerClient)null!).AsIHostedFileClient()); + } + + [Fact] + public void AsIHostedFileClient_OpenAIClient_ProducesExpectedMetadata() + { + Uri endpoint = new("http://localhost/some/endpoint"); + var client = new OpenAIClient(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); + + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + var metadata = fileClient.GetService(); + + Assert.NotNull(metadata); + Assert.Equal("openai", metadata.ProviderName); + Assert.Equal(endpoint, metadata.ProviderUri); + } + + [Fact] + public void AsIHostedFileClient_OpenAIFileClient_ProducesExpectedMetadata() + { + Uri endpoint = new("http://localhost/some/endpoint"); + var client = new OpenAIClient(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); + + using IHostedFileClient fileClient = client.GetOpenAIFileClient().AsIHostedFileClient(); + var metadata = fileClient.GetService(); + + Assert.NotNull(metadata); + Assert.Equal("openai", metadata.ProviderName); + Assert.Equal(endpoint, metadata.ProviderUri); + } + + [Fact] + public void AsIHostedFileClient_ContainerClient_ProducesExpectedMetadata() + { + Uri endpoint = new("http://localhost/some/endpoint"); + var client = new OpenAIClient(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); + + using IHostedFileClient fileClient = client.GetContainerClient().AsIHostedFileClient(); + var metadata = fileClient.GetService(); + + Assert.NotNull(metadata); + Assert.Equal("openai", metadata.ProviderName); + Assert.Equal(endpoint, metadata.ProviderUri); + } + + [Fact] + public async Task Upload_DefaultPurpose_SendsAssistants() + { + const string ExpectedInput = """ + { + "purpose": "assistants" + } + """; + + const string Output = """ + { + "id": "file-abc123", + "object": "file", + "bytes": 140, + "created_at": 1613677385, + "filename": "mydata.jsonl", + "purpose": "assistants" + } + """; + + using VerbatimMultiPartHttpHandler handler = new(ExpectedInput, Output) { ExpectedRequestUriContains = "files" }; + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var stream = new MemoryStream(new byte[] { 0x01, 0x02, 0x03 }); + var result = await client.UploadAsync(stream); + + Assert.NotNull(result); + Assert.Equal("file-abc123", result.Id); + Assert.Equal("mydata.jsonl", result.Name); + Assert.Equal("Assistants", result.Purpose); + Assert.Equal(140, result.SizeInBytes); + Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1_613_677_385), result.CreatedAt); + Assert.Equal("application/x-ndjson", result.MediaType); + Assert.NotNull(result.RawRepresentation); + } + + [Fact] + public async Task Upload_ExplicitPurposeFineTune_SendsFineTune() + { + const string ExpectedInput = """ + { + "purpose": "fine-tune" + } + """; + + const string Output = """ + { + "id": "file-def456", + "object": "file", + "bytes": 200, + "created_at": 1613677400, + "filename": "training.jsonl", + "purpose": "fine-tune" + } + """; + + using VerbatimMultiPartHttpHandler handler = new(ExpectedInput, Output) { ExpectedRequestUriContains = "files" }; + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var stream = new MemoryStream(new byte[] { 0x01, 0x02 }); + var result = await client.UploadAsync(stream, options: new HostedFileUploadOptions { Purpose = "fine-tune" }); + + Assert.NotNull(result); + Assert.Equal("file-def456", result.Id); + Assert.Equal("FineTune", result.Purpose); + } + + [Fact] + public async Task Upload_NullContent_ThrowsArgumentNullException() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("content", () => client.UploadAsync(null!)); + } + + [Fact] + public async Task Download_NullFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.DownloadAsync(null!)); + } + + [Fact] + public async Task Download_WhitespaceFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.DownloadAsync(" ")); + } + + [Fact] + public async Task GetFileInfo_ReturnsCorrectHostedFile() + { + const string Output = """ + { + "id": "file-abc123", + "object": "file", + "bytes": 140, + "created_at": 1613677385, + "filename": "mydata.jsonl", + "purpose": "assistants" + } + """; + + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput { Body = null }, Output); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.GetFileInfoAsync("file-abc123"); + + Assert.NotNull(result); + Assert.Equal("file-abc123", result.Id); + Assert.Equal("mydata.jsonl", result.Name); + Assert.Equal(140, result.SizeInBytes); + Assert.Equal("Assistants", result.Purpose); + Assert.Equal(DateTimeOffset.FromUnixTimeSeconds(1_613_677_385), result.CreatedAt); + Assert.Equal("application/x-ndjson", result.MediaType); + Assert.NotNull(result.RawRepresentation); + } + + [Fact] + public async Task GetFileInfo_NotFound_ReturnsNull() + { + using NotFoundHandler handler = new(); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.GetFileInfoAsync("file-nonexistent"); + + Assert.Null(result); + } + + [Fact] + public async Task GetFileInfo_NullFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.GetFileInfoAsync(null!)); + } + + [Fact] + public async Task GetFileInfo_WhitespaceFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.GetFileInfoAsync(" ")); + } + + [Fact] + public async Task ListFiles_ReturnsAllFiles() + { + const string Output = """ + { + "data": [ + {"id": "file-abc123", "object": "file", "bytes": 140, "created_at": 1613677385, "filename": "a.jsonl", "purpose": "assistants"}, + {"id": "file-def456", "object": "file", "bytes": 200, "created_at": 1613677400, "filename": "b.jsonl", "purpose": "fine-tune"} + ], + "object": "list" + } + """; + + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput { Body = null }, Output); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var files = await CollectAsync(client.ListFilesAsync()); + + Assert.Equal(2, files.Count); + + Assert.Equal("file-abc123", files[0].Id); + Assert.Equal("a.jsonl", files[0].Name); + Assert.Equal(140, files[0].SizeInBytes); + Assert.Equal("Assistants", files[0].Purpose); + Assert.Equal("application/x-ndjson", files[0].MediaType); + Assert.NotNull(files[0].RawRepresentation); + + Assert.Equal("file-def456", files[1].Id); + Assert.Equal("b.jsonl", files[1].Name); + Assert.Equal(200, files[1].SizeInBytes); + Assert.Equal("FineTune", files[1].Purpose); + Assert.Equal("application/x-ndjson", files[1].MediaType); + Assert.NotNull(files[1].RawRepresentation); + } + + [Fact] + public async Task ListFiles_WithLimit_ReturnsLimitedCount() + { + const string Output = """ + { + "data": [ + {"id": "file-abc123", "object": "file", "bytes": 140, "created_at": 1613677385, "filename": "a.jsonl", "purpose": "assistants"}, + {"id": "file-def456", "object": "file", "bytes": 200, "created_at": 1613677400, "filename": "b.jsonl", "purpose": "fine-tune"}, + {"id": "file-ghi789", "object": "file", "bytes": 300, "created_at": 1613677500, "filename": "c.jsonl", "purpose": "assistants"} + ], + "object": "list" + } + """; + + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput { Body = null }, Output); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var files = await CollectAsync(client.ListFilesAsync(new HostedFileListOptions { Limit = 2 })); + + Assert.Equal(2, files.Count); + } + + [Fact] + public async Task ListFiles_WithPurposeFilter() + { + const string Output = """ + { + "data": [ + {"id": "file-abc123", "object": "file", "bytes": 140, "created_at": 1613677385, "filename": "a.jsonl", "purpose": "assistants"} + ], + "object": "list" + } + """; + + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput { Body = null }, Output); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var files = await CollectAsync(client.ListFilesAsync(new HostedFileListOptions { Purpose = "assistants" })); + + Assert.Single(files); + Assert.Equal("file-abc123", files[0].Id); + Assert.Equal("Assistants", files[0].Purpose); + } + + [Fact] + public async Task Delete_Success_ReturnsTrue() + { + const string Output = """ + { + "id": "file-abc123", + "object": "file", + "deleted": true + } + """; + + using VerbatimHttpHandler handler = new(new HttpHandlerExpectedInput { Body = null }, Output); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.DeleteAsync("file-abc123"); + + Assert.True(result); + } + + [Fact] + public async Task Delete_NotFound_ReturnsFalse() + { + using NotFoundHandler handler = new(); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.DeleteAsync("file-nonexistent"); + + Assert.False(result); + } + + [Fact] + public async Task Delete_NullFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.DeleteAsync(null!)); + } + + [Fact] + public async Task Delete_WhitespaceFileId_Throws() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileClient(httpClient); + + await Assert.ThrowsAsync("fileId", () => client.DeleteAsync(" ")); + } + + [Fact] + public async Task Upload_WithScope_OnFileOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileOnlyClient(httpClient); + + using var stream = new MemoryStream(new byte[] { 0x01 }); + await Assert.ThrowsAsync( + () => client.UploadAsync(stream, options: new HostedFileUploadOptions { Scope = "container-123" })); + } + + [Fact] + public async Task Download_WithScope_OnFileOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileOnlyClient(httpClient); + + await Assert.ThrowsAsync( + () => client.DownloadAsync("file-abc123", new HostedFileDownloadOptions { Scope = "container-123" })); + } + + [Fact] + public async Task GetFileInfo_WithScope_OnFileOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileOnlyClient(httpClient); + + await Assert.ThrowsAsync( + () => client.GetFileInfoAsync("file-abc123", new HostedFileGetOptions { Scope = "container-123" })); + } + + [Fact] + public async Task ListFiles_WithScope_OnFileOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileOnlyClient(httpClient); + + await Assert.ThrowsAsync( + async () => await CollectAsync(client.ListFilesAsync(new HostedFileListOptions { Scope = "container-123" }))); + } + + [Fact] + public async Task Delete_WithScope_OnFileOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateFileOnlyClient(httpClient); + + await Assert.ThrowsAsync( + () => client.DeleteAsync("file-abc123", new HostedFileDeleteOptions { Scope = "container-123" })); + } + + [Fact] + public async Task Upload_WithoutScope_OnContainerOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateContainerClient(httpClient); + + using var stream = new MemoryStream(new byte[] { 0x01 }); + await Assert.ThrowsAsync( + () => client.UploadAsync(stream)); + } + + [Fact] + public async Task Download_WithoutScope_OnContainerOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateContainerClient(httpClient); + + await Assert.ThrowsAsync( + () => client.DownloadAsync("file-abc123")); + } + + [Fact] + public async Task GetFileInfo_WithoutScope_OnContainerOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateContainerClient(httpClient); + + await Assert.ThrowsAsync( + () => client.GetFileInfoAsync("file-abc123")); + } + + [Fact] + public async Task ListFiles_WithoutScope_OnContainerOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateContainerClient(httpClient); + + await Assert.ThrowsAsync( + async () => await CollectAsync(client.ListFilesAsync())); + } + + [Fact] + public async Task Delete_WithoutScope_OnContainerOnlyClient_ThrowsInvalidOperation() + { + using HttpClient httpClient = new(); + using IHostedFileClient client = CreateContainerClient(httpClient); + + await Assert.ThrowsAsync( + () => client.DeleteAsync("file-abc123")); + } + + [Fact] + public void GetService_ReturnsMetadata() + { + var client = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + + var metadata = fileClient.GetService(); + + Assert.NotNull(metadata); + Assert.Equal("openai", metadata.ProviderName); + } + + [Fact] + public void GetService_ReturnsSelfForIHostedFileClient() + { + var client = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + + var self = fileClient.GetService(); + + Assert.Same(fileClient, self); + } + + [Fact] + public void GetService_ReturnsOpenAIFileClient() + { + var openAIClient = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = openAIClient.AsIHostedFileClient(); + + var innerClient = fileClient.GetService(); + + Assert.NotNull(innerClient); + } + + [Fact] + public void GetService_ReturnsContainerClient() + { + var openAIClient = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = openAIClient.AsIHostedFileClient(); + + var innerClient = fileClient.GetService(); + + Assert.NotNull(innerClient); + } + + [Fact] + public void GetService_ReturnsNullForUnknownType() + { + var client = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + + var result = fileClient.GetService(typeof(string)); + + Assert.Null(result); + } + + [Fact] + public void GetService_WithNonNullKey_ReturnsNull() + { + var client = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + + var result = fileClient.GetService(typeof(HostedFileClientMetadata), "somekey"); + + Assert.Null(result); + } + + [Fact] + public void GetService_FileOnlyClient_ReturnsNullForContainerClient() + { + var openAIClient = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = openAIClient.GetOpenAIFileClient().AsIHostedFileClient(); + + var innerClient = fileClient.GetService(); + + Assert.Null(innerClient); + } + + [Fact] + public void GetService_ContainerOnlyClient_ReturnsNullForOpenAIFileClient() + { + var openAIClient = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = openAIClient.GetContainerClient().AsIHostedFileClient(); + + var innerClient = fileClient.GetService(); + + Assert.Null(innerClient); + } + + [Fact] + public async Task Download_ReturnsStreamWithMediaTypeAndFileName() + { + byte[] fileData = "Hello, World!"u8.ToArray(); + + using var handler = new RoutingHandler(request => + request.RequestUri!.AbsolutePath.EndsWith("/content", StringComparison.Ordinal) + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(fileData) } + : new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "file-abc123", + "object": "file", + "bytes": 13, + "created_at": 1613677385, + "filename": "hello.txt", + "purpose": "assistants" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var stream = await client.DownloadAsync("file-abc123"); + + Assert.Equal("text/plain", stream.MediaType); + Assert.Equal("hello.txt", stream.FileName); + + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms); + Assert.Equal(fileData, ms.ToArray()); + } + + [Fact] + public async Task Download_StreamWriteThrowsNotSupportedException() + { + byte[] fileData = "Hello, World!"u8.ToArray(); + + using var handler = new RoutingHandler(request => + request.RequestUri!.AbsolutePath.EndsWith("/content", StringComparison.Ordinal) + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(fileData) } + : new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "file-abc123", + "object": "file", + "bytes": 13, + "created_at": 1613677385, + "filename": "hello.txt", + "purpose": "assistants" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var stream = await client.DownloadAsync("file-abc123"); + + Assert.False(stream.CanWrite); + Assert.Throws(() => stream.Write(new byte[1], 0, 1)); + await Assert.ThrowsAsync(() => stream.WriteAsync(new byte[1], 0, 1)); +#if NET + Assert.Throws(() => stream.Write(new ReadOnlySpan(new byte[1]))); + await Assert.ThrowsAsync(async () => await stream.WriteAsync(new ReadOnlyMemory(new byte[1]))); +#endif + } + + [Fact] + public void GetService_NullServiceType_Throws() + { + var client = new OpenAIClient(new ApiKeyCredential("key")); + using IHostedFileClient fileClient = client.AsIHostedFileClient(); + + Assert.Throws("serviceType", () => fileClient.GetService(null!)); + } + + [Fact] + public async Task Upload_WithScope_UsesContainerApi() + { + using var handler = new RoutingHandler(request => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "cfile-abc", + "object": "container.file", + "path": "uploads/data.csv" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var contentStream = new MemoryStream(new byte[] { 1, 2, 3 }); + var result = await client.UploadAsync(contentStream, "text/csv", "data.csv", new HostedFileUploadOptions { Scope = "ctr-123" }); + + Assert.Equal("cfile-abc", result.Id); + Assert.Equal("data.csv", result.Name); + Assert.Equal("text/csv", result.MediaType); + } + + [Fact] + public async Task Upload_WithScope_DoesNotDisposeCallerStream() + { + using var handler = new RoutingHandler(request => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "cfile-abc", + "object": "container.file", + "path": "uploads/data.csv" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var contentStream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.UploadAsync(contentStream, "text/csv", "data.csv", new HostedFileUploadOptions { Scope = "ctr-123" }); + + // The caller's stream should still be usable after upload completes. + contentStream.Position = 0; + Assert.Equal(3, contentStream.Length); + Assert.Equal(1, contentStream.ReadByte()); + } + + [Fact] + public async Task Download_WithScope_UsesContainerApi() + { + byte[] fileData = "container content"u8.ToArray(); + + using var handler = new RoutingHandler(request => + request.RequestUri!.AbsolutePath.EndsWith("/content", StringComparison.Ordinal) + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(fileData) } + : new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "cfile-1", + "object": "container.file", + "path": "/uploads/data.txt" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + using var stream = await client.DownloadAsync("cfile-1", new HostedFileDownloadOptions { Scope = "ctr-123" }); + + Assert.Equal("text/plain", stream.MediaType); + Assert.Equal("data.txt", stream.FileName); + + using var ms = new MemoryStream(); + await stream.CopyToAsync(ms); + Assert.Equal(fileData, ms.ToArray()); + } + + [Fact] + public async Task GetFileInfo_WithScope_ReturnsContainerFileInfo() + { + using var handler = new RoutingHandler(request => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "cfile-1", + "object": "container.file", + "path": "/path/to/report.pdf" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.GetFileInfoAsync("cfile-1", new HostedFileGetOptions { Scope = "ctr-123" }); + + Assert.NotNull(result); + Assert.Equal("cfile-1", result.Id); + Assert.Equal("report.pdf", result.Name); + Assert.Equal("application/pdf", result.MediaType); + Assert.NotNull(result.RawRepresentation); + } + + [Fact] + public async Task GetFileInfo_WithScope_NotFound_ReturnsNull() + { + using NotFoundHandler handler = new(); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.GetFileInfoAsync("cfile-nonexistent", new HostedFileGetOptions { Scope = "ctr-123" }); + + Assert.Null(result); + } + + [Fact] + public async Task ListFiles_WithScope_ReturnsContainerFiles() + { + using var handler = new RoutingHandler(request => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "data": [ + {"id": "cfile-1", "object": "container.file", "path": "file1.txt"}, + {"id": "cfile-2", "object": "container.file", "path": "dir/file2.csv"} + ], + "object": "list", + "has_more": false, + "first_id": "cfile-1", + "last_id": "cfile-2" + } + """, Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var files = await CollectAsync(client.ListFilesAsync(new HostedFileListOptions { Scope = "ctr-123" })); + + Assert.Equal(2, files.Count); + + Assert.Equal("cfile-1", files[0].Id); + Assert.Equal("file1.txt", files[0].Name); + Assert.Equal("text/plain", files[0].MediaType); + Assert.NotNull(files[0].RawRepresentation); + + Assert.Equal("cfile-2", files[1].Id); + Assert.Equal("file2.csv", files[1].Name); + Assert.Equal("text/csv", files[1].MediaType); + Assert.NotNull(files[1].RawRepresentation); + } + + [Fact] + public async Task Delete_WithScope_Success_ReturnsTrue() + { + using var handler = new RoutingHandler(request => + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json") + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.DeleteAsync("cfile-1", new HostedFileDeleteOptions { Scope = "ctr-123" }); + + Assert.True(result); + } + + [Fact] + public async Task Delete_WithScope_NotFound_ReturnsFalse() + { + using NotFoundHandler handler = new(); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateFileClient(httpClient); + + var result = await client.DeleteAsync("cfile-nonexistent", new HostedFileDeleteOptions { Scope = "ctr-123" }); + + Assert.False(result); + } + + [Fact] + public async Task DefaultScope_IsUsedWhenNoScopeInOptions() + { + using var handler = new RoutingHandler(request => + { + Assert.Contains("/containers/default-ctr/", request.RequestUri!.AbsolutePath); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + { + "id": "cfile-1", + "object": "container.file", + "path": "test.txt" + } + """, Encoding.UTF8, "application/json") + }; + }); + using HttpClient httpClient = new(handler); + using IHostedFileClient client = CreateContainerClient(httpClient, defaultScope: "default-ctr"); + + var result = await client.GetFileInfoAsync("cfile-1"); + + Assert.NotNull(result); + Assert.Equal("cfile-1", result.Id); + Assert.Equal("test.txt", result.Name); + } + + private static IHostedFileClient CreateFileClient(HttpClient httpClient) => + new OpenAIClient(new ApiKeyCredential("apikey"), new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) + .AsIHostedFileClient(); + + private static IHostedFileClient CreateFileOnlyClient(HttpClient httpClient) => + new OpenAIClient(new ApiKeyCredential("apikey"), new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) + .GetOpenAIFileClient() + .AsIHostedFileClient(); + + private static IHostedFileClient CreateContainerClient(HttpClient httpClient, string? defaultScope = null) => + new OpenAIClient(new ApiKeyCredential("apikey"), new OpenAIClientOptions { Transport = new HttpClientPipelineTransport(httpClient) }) + .GetContainerClient() + .AsIHostedFileClient(defaultScope); + + private sealed class NotFoundHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) { Content = new StringContent("{}") }); + } + + private sealed class RoutingHandler(Func handler) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(handler(request)); + } + + private static async Task> CollectAsync(IAsyncEnumerable source) + { + var list = new List(); + await foreach (var item in source) + { + list.Add(item); + } + + return list; + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Files/LoggingHostedFileClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Files/LoggingHostedFileClientTests.cs new file mode 100644 index 00000000000..37d7f940a45 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Files/LoggingHostedFileClientTests.cs @@ -0,0 +1,567 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Testing; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class LoggingHostedFileClientTests +{ + [Fact] + public void LoggingHostedFileClient_InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new LoggingHostedFileClient(null!, NullLogger.Instance)); + Assert.Throws("logger", () => new LoggingHostedFileClient(new TestHostedFileClient(), null!)); + } + + [Fact] + public void UseLogging_AvoidsInjectingNopClient() + { + using var innerClient = new TestHostedFileClient(); + + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingHostedFileClient))); + Assert.Same(innerClient, innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IHostedFileClient))); + + using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); + Assert.NotNull(innerClient.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingHostedFileClient))); + + ServiceCollection c = new(); + c.AddFakeLogging(); + var services = c.BuildServiceProvider(); + Assert.NotNull(innerClient.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingHostedFileClient))); + Assert.NotNull(innerClient.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingHostedFileClient))); + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingHostedFileClient))); + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task UploadAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + + ServiceCollection c = new(); + c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + var services = c.BuildServiceProvider(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-123") { Name = "test.txt" }), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging() + .Build(services); + + using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.UploadAsync(stream, "text/plain", "test.txt", new HostedFileUploadOptions { Purpose = "assistants" }); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("UploadAsync invoked.", entry.Message); + Assert.Contains("text/plain", entry.Message); + Assert.Contains("test.txt", entry.Message); + }, + entry => + { + Assert.Contains("UploadAsync completed:", entry.Message); + Assert.Contains("file-123", entry.Message); + }); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("UploadAsync invoked.", entry.Message); + Assert.DoesNotContain("text/plain", entry.Message); + Assert.DoesNotContain("test.txt", entry.Message); + }, + entry => + { + Assert.Contains("UploadAsync completed.", entry.Message); + Assert.DoesNotContain("file-123", entry.Message); + }); + } + else + { + Assert.Empty(logs); + } + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task DownloadAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + + using var innerClient = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + Task.FromResult(new TestDownloadStream(new byte[] { 1 })), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + using var stream = await client.DownloadAsync("file-123"); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("DownloadAsync invoked.", entry.Message); + Assert.Contains("file-123", entry.Message); + }, + entry => Assert.Contains("DownloadAsync completed.", entry.Message)); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("DownloadAsync invoked.", entry.Message); + Assert.DoesNotContain("file-123", entry.Message); + }, + entry => Assert.Contains("DownloadAsync completed.", entry.Message)); + } + else + { + Assert.Empty(logs); + } + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task GetFileInfoAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + + using var innerClient = new TestHostedFileClient + { + GetFileInfoAsyncCallback = (fileId, options, ct) => + Task.FromResult(new HostedFile("file-456") { Name = "report.pdf" }), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await client.GetFileInfoAsync("file-456"); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("GetFileInfoAsync invoked.", entry.Message); + Assert.Contains("file-456", entry.Message); + }, + entry => + { + Assert.Contains("GetFileInfoAsync completed:", entry.Message); + Assert.Contains("file-456", entry.Message); + Assert.Contains("report.pdf", entry.Message); + }); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("GetFileInfoAsync invoked.", entry.Message); + Assert.DoesNotContain("file-456", entry.Message); + }, + entry => + { + Assert.Contains("GetFileInfoAsync completed.", entry.Message); + Assert.DoesNotContain("report.pdf", entry.Message); + }); + } + else + { + Assert.Empty(logs); + } + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task ListFilesAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + + using var innerClient = new TestHostedFileClient + { + ListFilesAsyncCallback = (options, ct) => GetFilesAsync(), + }; + + static async IAsyncEnumerable GetFilesAsync() + { + await Task.Yield(); + yield return new HostedFile("file-1") { Name = "a.txt" }; + yield return new HostedFile("file-2") { Name = "b.txt" }; + } + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await foreach (var file in client.ListFilesAsync()) + { + // consume + } + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => Assert.Contains("ListFilesAsync invoked.", entry.Message), + entry => + { + Assert.Contains("ListFilesAsync received item:", entry.Message); + Assert.Contains("file-1", entry.Message); + }, + entry => + { + Assert.Contains("ListFilesAsync received item:", entry.Message); + Assert.Contains("file-2", entry.Message); + }, + entry => Assert.Contains("ListFilesAsync completed.", entry.Message)); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => Assert.Contains("ListFilesAsync invoked.", entry.Message), + entry => Assert.Contains("ListFilesAsync completed.", entry.Message)); + } + else + { + Assert.Empty(logs); + } + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task DeleteAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + + using var innerClient = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, ct) => Task.FromResult(true), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await client.DeleteAsync("file-789"); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("DeleteAsync invoked.", entry.Message); + Assert.Contains("file-789", entry.Message); + }, + entry => + { + Assert.Contains("DeleteAsync completed:", entry.Message); + Assert.Contains("true", entry.Message); + }); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => + { + Assert.Contains("DeleteAsync invoked.", entry.Message); + Assert.DoesNotContain("file-789", entry.Message); + }, + entry => Assert.Contains("DeleteAsync completed.", entry.Message)); + } + else + { + Assert.Empty(logs); + } + } + + [Fact] + public async Task UploadAsync_OnException_LogsError() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + throw new InvalidOperationException("upload failed"), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await Assert.ThrowsAsync(() => client.UploadAsync(stream)); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("UploadAsync invoked.", entry.Message), + entry => + { + Assert.Contains("UploadAsync failed.", entry.Message); + Assert.Equal(LogLevel.Error, entry.Level); + }); + } + + [Fact] + public async Task UploadAsync_OnCancellation_LogsCanceled() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + throw new OperationCanceledException(), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await Assert.ThrowsAsync(() => client.UploadAsync(stream)); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("UploadAsync invoked.", entry.Message), + entry => Assert.Contains("UploadAsync canceled.", entry.Message)); + } + + [Fact] + public async Task DownloadAsync_OnException_LogsError() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("download failed"), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await Assert.ThrowsAsync(() => client.DownloadAsync("file-1")); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("DownloadAsync invoked.", entry.Message), + entry => + { + Assert.Contains("DownloadAsync failed.", entry.Message); + Assert.Equal(LogLevel.Error, entry.Level); + }); + } + + [Fact] + public async Task GetFileInfoAsync_OnException_LogsError() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + GetFileInfoAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("get failed"), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await Assert.ThrowsAsync(() => client.GetFileInfoAsync("file-1")); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("GetFileInfoAsync invoked.", entry.Message), + entry => + { + Assert.Contains("GetFileInfoAsync failed.", entry.Message); + Assert.Equal(LogLevel.Error, entry.Level); + }); + } + + [Fact] + public async Task DeleteAsync_OnException_LogsError() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("delete failed"), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await Assert.ThrowsAsync(() => client.DeleteAsync("file-1")); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("DeleteAsync invoked.", entry.Message), + entry => + { + Assert.Contains("DeleteAsync failed.", entry.Message); + Assert.Equal(LogLevel.Error, entry.Level); + }); + } + + [Fact] + public async Task DeleteAsync_OnCancellation_LogsCanceled() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, ct) => + throw new OperationCanceledException(), + }; + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await Assert.ThrowsAsync(() => client.DeleteAsync("file-1")); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("DeleteAsync invoked.", entry.Message), + entry => Assert.Contains("DeleteAsync canceled.", entry.Message)); + } + + [Fact] + public async Task ListFilesAsync_OnIterationException_LogsError() + { + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); + + using var innerClient = new TestHostedFileClient + { + ListFilesAsyncCallback = (options, ct) => ThrowOnSecondItem(), + }; + + static async IAsyncEnumerable ThrowOnSecondItem() + { + await Task.Yield(); + yield return new HostedFile("file-1"); + throw new InvalidOperationException("iteration failed"); + } + + using IHostedFileClient client = innerClient + .AsBuilder() + .UseLogging(loggerFactory) + .Build(); + + await Assert.ThrowsAsync(async () => + { + await foreach (var file in client.ListFilesAsync()) + { + _ = file; + } + }); + + var logs = collector.GetSnapshot(); + Assert.Collection(logs, + entry => Assert.Contains("ListFilesAsync invoked.", entry.Message), + entry => + { + Assert.Contains("ListFilesAsync failed.", entry.Message); + Assert.Equal(LogLevel.Error, entry.Level); + }); + } + + private sealed class TestDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public TestDownloadStream(byte[] data) + { + _inner = new MemoryStream(data); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Files/OpenTelemetryHostedFileClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Files/OpenTelemetryHostedFileClientTests.cs new file mode 100644 index 00000000000..04bd88de8db --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Files/OpenTelemetryHostedFileClientTests.cs @@ -0,0 +1,609 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using OpenTelemetry.Trace; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OpenTelemetryHostedFileClientTests +{ + [Fact] + public void InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new OpenTelemetryHostedFileClient(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UploadAsync_TracesExpectedData(bool enableSensitiveData) + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-abc") { Name = "test.txt", SizeInBytes = 1024 }), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName, configure: c => c.EnableSensitiveData = enableSensitiveData) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.UploadAsync(stream, "text/plain", "test.txt", new HostedFileUploadOptions { Purpose = "assistants", Scope = "container-1" }); + + var activity = Assert.Single(activities); + Assert.Equal("files.upload", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + Assert.Equal("files.upload", activity.GetTagItem("files.operation.name")); + Assert.Equal("testprovider", activity.GetTagItem("files.provider.name")); + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(8080, (int)activity.GetTagItem("server.port")!); + Assert.True(activity.Duration.TotalMilliseconds > 0); + Assert.Equal(ActivityStatusCode.Unset, activity.Status); + + // Always-present operational metadata + Assert.Equal("text/plain", activity.GetTagItem("files.media_type")); + Assert.Equal("assistants", activity.GetTagItem("files.purpose")); + Assert.Equal("container-1", activity.GetTagItem("files.scope")); + Assert.Equal("file-abc", activity.GetTagItem("files.id")); + Assert.Equal(1024L, activity.GetTagItem("file.size")); + + // file.name is sensitive (could contain PII) + if (enableSensitiveData) + { + Assert.Equal("test.txt", activity.GetTagItem("file.name")); + } + else + { + Assert.Null(activity.GetTagItem("file.name")); + } + } + + [Fact] + public async Task DownloadAsync_TracesExpectedData() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + Task.FromResult(new TestDownloadStream(new byte[] { 1 })), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + using var stream = await client.DownloadAsync("file-xyz", new HostedFileDownloadOptions { Scope = "container-2" }); + + var activity = Assert.Single(activities); + Assert.Equal("files.download", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + Assert.Equal("files.download", activity.GetTagItem("files.operation.name")); + Assert.Equal("testprovider", activity.GetTagItem("files.provider.name")); + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(8080, (int)activity.GetTagItem("server.port")!); + Assert.True(activity.Duration.TotalMilliseconds > 0); + Assert.Equal(ActivityStatusCode.Unset, activity.Status); + + Assert.Equal("file-xyz", activity.GetTagItem("files.id")); + Assert.Equal("container-2", activity.GetTagItem("files.scope")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetFileInfoAsync_TracesExpectedData(bool enableSensitiveData) + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + GetFileInfoAsyncCallback = (fileId, options, ct) => + Task.FromResult(new HostedFile("file-info") { Name = "report.pdf", SizeInBytes = 2048 }), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName, configure: c => c.EnableSensitiveData = enableSensitiveData) + .Build(); + + await client.GetFileInfoAsync("file-info", new HostedFileGetOptions { Scope = "container-3" }); + + var activity = Assert.Single(activities); + Assert.Equal("files.get_info", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + Assert.Equal("files.get_info", activity.GetTagItem("files.operation.name")); + Assert.Equal("testprovider", activity.GetTagItem("files.provider.name")); + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(8080, (int)activity.GetTagItem("server.port")!); + Assert.True(activity.Duration.TotalMilliseconds > 0); + Assert.Equal(ActivityStatusCode.Unset, activity.Status); + + Assert.Equal("file-info", activity.GetTagItem("files.id")); + Assert.Equal("container-3", activity.GetTagItem("files.scope")); + Assert.Equal(2048L, activity.GetTagItem("file.size")); + + if (enableSensitiveData) + { + Assert.Equal("report.pdf", activity.GetTagItem("file.name")); + } + else + { + Assert.Null(activity.GetTagItem("file.name")); + } + } + + [Fact] + public async Task ListFilesAsync_TracesExpectedData() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + ListFilesAsyncCallback = (options, ct) => GetFilesAsync(), + GetServiceCallback = CreateMetadataCallback(), + }; + + static async IAsyncEnumerable GetFilesAsync() + { + await Task.Yield(); + yield return new HostedFile("file-1") { Name = "a.txt" }; + yield return new HostedFile("file-2") { Name = "b.txt" }; + yield return new HostedFile("file-3") { Name = "c.txt" }; + } + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await foreach (var file in client.ListFilesAsync(new HostedFileListOptions { Purpose = "assistants", Scope = "container-4" })) + { + _ = file; + } + + var activity = Assert.Single(activities); + Assert.Equal("files.list", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + Assert.Equal("files.list", activity.GetTagItem("files.operation.name")); + Assert.Equal("testprovider", activity.GetTagItem("files.provider.name")); + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(8080, (int)activity.GetTagItem("server.port")!); + Assert.True(activity.Duration.TotalMilliseconds > 0); + Assert.Equal(ActivityStatusCode.Unset, activity.Status); + Assert.Equal(3, activity.GetTagItem("files.list.count")); + + Assert.Equal("container-4", activity.GetTagItem("files.scope")); + Assert.Equal("assistants", activity.GetTagItem("files.purpose")); + } + + [Fact] + public async Task DeleteAsync_TracesExpectedData() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, ct) => Task.FromResult(true), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await client.DeleteAsync("file-del", new HostedFileDeleteOptions { Scope = "container-5" }); + + var activity = Assert.Single(activities); + Assert.Equal("files.delete", activity.DisplayName); + Assert.Equal(ActivityKind.Client, activity.Kind); + Assert.Equal("files.delete", activity.GetTagItem("files.operation.name")); + Assert.Equal("testprovider", activity.GetTagItem("files.provider.name")); + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(8080, (int)activity.GetTagItem("server.port")!); + Assert.True(activity.Duration.TotalMilliseconds > 0); + Assert.Equal(ActivityStatusCode.Unset, activity.Status); + + Assert.Equal("file-del", activity.GetTagItem("files.id")); + Assert.Equal("container-5", activity.GetTagItem("files.scope")); + } + + [Fact] + public async Task UploadAsync_OnError_SetsErrorStatus() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + throw new InvalidOperationException("upload failed"), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await Assert.ThrowsAsync(() => client.UploadAsync(stream)); + + var activity = Assert.Single(activities); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("upload failed", activity.StatusDescription); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + } + + [Fact] + public async Task ListFilesAsync_OnIterationError_SetsErrorStatus() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + ListFilesAsyncCallback = (options, ct) => ThrowOnSecondItem(), + GetServiceCallback = CreateMetadataCallback(), + }; + + static async IAsyncEnumerable ThrowOnSecondItem() + { + await Task.Yield(); + yield return new HostedFile("file-1"); + throw new InvalidOperationException("iteration failed"); + } + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await Assert.ThrowsAsync(async () => + { + await foreach (var file in client.ListFilesAsync()) + { + _ = file; + } + }); + + var activity = Assert.Single(activities); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + Assert.Equal(1, activity.GetTagItem("files.list.count")); + } + + [Fact] + public async Task GetService_ReturnsActivitySource() + { + var sourceName = Guid.NewGuid().ToString(); + + using var innerClient = new TestHostedFileClient(); + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + var activitySource = client.GetService(); + Assert.NotNull(activitySource); + Assert.Equal(sourceName, activitySource.Name); + } + + [Fact] + public async Task NoListeners_NoActivityCreated() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + + // Deliberately not subscribing to the source name — no listeners + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource("some-other-source") + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-1")), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await client.UploadAsync(stream); + + Assert.Empty(activities); + } + + [Fact] + public async Task DownloadAsync_OnError_SetsErrorStatus() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + DownloadAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("download failed"), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await Assert.ThrowsAsync(() => client.DownloadAsync("file-1")); + + var activity = Assert.Single(activities); + Assert.Equal("files.download", activity.DisplayName); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("download failed", activity.StatusDescription); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + } + + [Fact] + public async Task DeleteAsync_OnError_SetsErrorStatus() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + DeleteAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("delete failed"), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await Assert.ThrowsAsync(() => client.DeleteAsync("file-1")); + + var activity = Assert.Single(activities); + Assert.Equal("files.delete", activity.DisplayName); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("delete failed", activity.StatusDescription); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + } + + [Fact] + public async Task GetFileInfoAsync_OnError_SetsErrorStatus() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + GetFileInfoAsyncCallback = (fileId, options, ct) => + throw new InvalidOperationException("get info failed"), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + await Assert.ThrowsAsync(() => client.GetFileInfoAsync("file-1")); + + var activity = Assert.Single(activities); + Assert.Equal("files.get_info", activity.DisplayName); + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal("get info failed", activity.StatusDescription); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + } + + [Fact] + public async Task NoMetadata_ServerTagsAbsent() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-1")), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await client.UploadAsync(stream); + + var activity = Assert.Single(activities); + Assert.Equal("files.upload", activity.DisplayName); + Assert.Null(activity.GetTagItem("files.provider.name")); + Assert.Null(activity.GetTagItem("server.address")); + Assert.Null(activity.GetTagItem("server.port")); + } + + [Fact] + public void UseOpenTelemetry_NullBuilder_Throws() + { + Assert.Throws("builder", + () => OpenTelemetryHostedFileClientBuilderExtensions.UseOpenTelemetry(null!)); + } + + [Fact] + public async Task AdditionalProperties_TaggedWhenSensitiveDataEnabled() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-1")), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName, configure: c => c.EnableSensitiveData = true) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await client.UploadAsync(stream, options: new HostedFileUploadOptions + { + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["custom.tag1"] = "value1", + ["custom.tag2"] = 42, + } + }); + + var activity = Assert.Single(activities); + Assert.Equal("value1", activity.GetTagItem("custom.tag1")); + Assert.Equal(42, activity.GetTagItem("custom.tag2")); + } + + [Fact] + public async Task AdditionalProperties_NotTaggedWhenSensitiveDataDisabled() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestHostedFileClient + { + UploadAsyncCallback = (stream, mediaType, fileName, options, ct) => + Task.FromResult(new HostedFile("file-1")), + GetServiceCallback = CreateMetadataCallback(), + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(sourceName: sourceName, configure: c => c.EnableSensitiveData = false) + .Build(); + + using var stream = new MemoryStream(new byte[] { 1 }); + await client.UploadAsync(stream, options: new HostedFileUploadOptions + { + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["custom.tag1"] = "value1", + } + }); + + var activity = Assert.Single(activities); + Assert.Null(activity.GetTagItem("custom.tag1")); + } + + private static Func CreateMetadataCallback() => + (serviceType, serviceKey) => + serviceType == typeof(HostedFileClientMetadata) ? new HostedFileClientMetadata("testprovider", new Uri("http://localhost:8080/files")) : + null; + + private sealed class TestDownloadStream : HostedFileDownloadStream + { + private readonly MemoryStream _inner; + + public TestDownloadStream(byte[] data) + { + _inner = new MemoryStream(data); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + public override void Flush() => _inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj index f70d9dbc75f..3c266cc872d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj @@ -21,6 +21,7 @@ +