Add HostedFile/VectorStoreContent, HostedFileSearchTool, and HostedCodeInterpreterTool.Inputs (#6620)

* Add HostedFileContent, HostedVectorStoreContent, HostedFileSearchTool, and HostedCodeInterpreterTool.Inputs

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -6,6 +6,10 @@
 - Added `ChatMessage.CreatedAt` so that chat messages can carry their timestamp.
 - Added a `[Description(...)]` attribute to `DataContent.Uri` to clarify its purpose when used in schemas.
 - Added `DataContent.Name` property to associate a name with the binary data, like a filename.
+- Added `HostedFileContent` for representing files hosted by the service.
+- Added `HostedVectorStoreContent` for representing vector stores hosted by the service.
+- Added `HostedFileSearchTool` to represent server-side file search tools.
+- Added `HostedCodeInterpreterTool.Inputs` to supply context about what state is available to the code interpreter tool.
 - Improved handling of function parameter data annotation attributes in `AIJsonUtilities.CreateJsonSchema`.
 - Fixed schema generation to include an items keyword for arrays of objects in `AIJsonUtilities.CreateJsonSchema`.
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatMessage.cs

@@ -7,6 +7,8 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json.Serialization;
 
+#pragma warning disable S3358 // Ternary operators should not be nested
+
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a chat message used by an <see cref="IChatClient" />.</summary>
@@ -107,7 +109,17 @@ public IList<AIContent> Contents
 
     /// <summary>Gets a <see cref="AIContent"/> object to display in the debugger display.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
-    private AIContent? ContentForDebuggerDisplay => _contents is { Count: > 0 } ? _contents[0] : null;
+    private AIContent? ContentForDebuggerDisplay
+    {
+        get
+        {
+            string text = Text;
+            return
+                !string.IsNullOrWhiteSpace(text) ? new TextContent(text) :
+                _contents is { Count: > 0 } ? _contents[0] :
+                null;
+        }
+    }
 
     /// <summary>Gets an indication for the debugger display of whether there's more content.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Text;
@@ -183,72 +184,106 @@ static async Task<ChatResponse> ToChatResponseAsync(
     }
 
     /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
-    internal static void CoalesceTextContent(List<AIContent> contents)
+    internal static void CoalesceTextContent(IList<AIContent> contents)
     {
-        Coalesce<TextContent>(contents, static text => new(text));
-        Coalesce<TextReasoningContent>(contents, static text => new(text));
+        Coalesce<TextContent>(contents, mergeSingle: false, static (contents, start, end) =>
+            new(MergeText(contents, start, end))
+            {
+                AdditionalProperties = contents[start].AdditionalProperties?.Clone()
+            });
 
-        // This implementation relies on TContent's ToString returning its exact text.
-        static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent> fromText)
-            where TContent : AIContent
+        Coalesce<TextReasoningContent>(contents, mergeSingle: false, static (contents, start, end) =>
+            new(MergeText(contents, start, end))
+            {
+                AdditionalProperties = contents[start].AdditionalProperties?.Clone()
+            });
+
+        static string MergeText(IList<AIContent> contents, int start, int end)
         {
-            StringBuilder? coalescedText = null;
+            StringBuilder sb = new();
+            for (int i = start; i < end; i++)
+            {
+                _ = sb.Append(contents[i]);
+            }
 
+            return sb.ToString();
+        }
+
+        static void Coalesce<TContent>(IList<AIContent> contents, bool mergeSingle, Func<IList<AIContent>, int, int, TContent> merge)
+            where TContent : AIContent
+        {
             // Iterate through all of the items in the list looking for contiguous items that can be coalesced.
             int start = 0;
-            while (start < contents.Count - 1)
+            while (start < contents.Count)
             {
-                // We need at least two TextContents in a row to be able to coalesce. We also avoid touching contents
-                // that have annotations, as we want to ensure the annotations (and in particular any start/end indices
-                // into the text content) remain accurate.
-                if (!TryAsCoalescable(contents[start], out var firstText))
+                if (!TryAsCoalescable(contents[start], out var firstContent))
                 {
                     start++;
                     continue;
                 }
 
-                if (!TryAsCoalescable(contents[start + 1], out var secondText))
+                // Iterate until we find a non-coalescable item.
+                int i = start + 1;
+                while (i < contents.Count && TryAsCoalescable(contents[i], out _))
                 {
-                    start += 2;
-                    continue;
+                    i++;
                 }
 
-                // Append the text from those nodes and continue appending subsequent TextContents until we run out.
-                // We null out nodes as their text is appended so that we can later remove them all in one O(N) operation.
-                coalescedText ??= new();
-                _ = coalescedText.Clear().Append(firstText).Append(secondText);
-                contents[start + 1] = null!;
-                int i = start + 2;
-                for (; i < contents.Count && TryAsCoalescable(contents[i], out TContent? next); i++)
+                // If there's only one item in the run, and we don't want to merge single items, skip it.
+                if (start == i - 1 && !mergeSingle)
                 {
-                    _ = coalescedText.Append(next);
-                    contents[i] = null!;
+                    start++;
+                    continue;
                 }
 
-                // Store the replacement node. We inherit the properties of the first text node. We don't
-                // currently propagate additional properties from the subsequent nodes. If we ever need to,
-                // we can add that here.
-                var newContent = fromText(coalescedText.ToString());
-                contents[start] = newContent;
-                newContent.AdditionalProperties = firstText.AdditionalProperties?.Clone();
+                // Store the replacement node and null out all of the nodes that we coalesced.
+                // We can then remove all coalesced nodes in one O(N) operation via RemoveAll.
+                // Leave start positioned at the start of the next run.
+                contents[start] = merge(contents, start, i);
 
-                start = i;
+                start++;
+                while (start < i)
+                {
+                    contents[start++] = null!;
+                }
 
                 static bool TryAsCoalescable(AIContent content, [NotNullWhen(true)] out TContent? coalescable)
                 {
-                    if (content is TContent && (content is not TextContent tc || tc.Annotations is not { Count: > 0 }))
+                    if (content is TContent tmp && tmp.Annotations is not { Count: > 0 })
                     {
-                        coalescable = (TContent)content;
+                        coalescable = tmp;
                         return true;
                     }
 
-                    coalescable = null!;
+                    coalescable = null;
                     return false;
                 }
             }
 
             // Remove all of the null slots left over from the coalescing process.
-            _ = contents.RemoveAll(u => u is null);
+            if (contents is List<AIContent> contentsList)
+            {
+                _ = contentsList.RemoveAll(u => u is null);
+            }
+            else
+            {
+                int nextSlot = 0;
+                int contentsCount = contents.Count;
+                for (int i = 0; i < contentsCount; i++)
+                {
+                    if (contents[i] is { } content)
+                    {
+                        contents[nextSlot++] = content;
+                    }
+                }
+
+                for (int i = contentsCount - 1; i >= nextSlot; i--)
+                {
+                    contents.RemoveAt(i);
+                }
+
+                Debug.Assert(nextSlot == contents.Count, "Expected final count to equal list length.");
+            }
         }
     }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs

@@ -7,6 +7,8 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json.Serialization;
 
+#pragma warning disable S3358 // Ternary operators should not be nested
+
 namespace Microsoft.Extensions.AI;
 
 /// <summary>
@@ -141,7 +143,17 @@ public IList<AIContent> Contents
 
     /// <summary>Gets a <see cref="AIContent"/> object to display in the debugger display.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
-    private AIContent? ContentForDebuggerDisplay => _contents is { Count: > 0 } ? _contents[0] : null;
+    private AIContent? ContentForDebuggerDisplay
+    {
+        get
+        {
+            string text = Text;
+            return
+                !string.IsNullOrWhiteSpace(text) ? new TextContent(text) :
+                _contents is { Count: > 0 } ? _contents[0] :
+                null;
+        }
+    }
 
     /// <summary>Gets an indication for the debugger display of whether there's more content.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -12,6 +12,8 @@ namespace Microsoft.Extensions.AI;
 [JsonDerivedType(typeof(ErrorContent), typeDiscriminator: "error")]
 [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: "functionCall")]
 [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: "functionResult")]
+[JsonDerivedType(typeof(HostedFileContent), typeDiscriminator: "hostedFile")]
+[JsonDerivedType(typeof(HostedVectorStoreContent), typeDiscriminator: "hostedVectorStore")]
 [JsonDerivedType(typeof(TextContent), typeDiscriminator: "text")]
 [JsonDerivedType(typeof(TextReasoningContent), typeDiscriminator: "reasoning")]
 [JsonDerivedType(typeof(UriContent), typeDiscriminator: "uri")]

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a file that is hosted by the AI service.
+/// </summary>
+/// <remarks>
+/// Unlike <see cref="DataContent"/> which contains the data for a file or blob, this class represents a file that is hosted
+/// by the AI service and referenced by an identifier. Such identifiers are specific to the provider.
+/// </remarks>
+[DebuggerDisplay("FileId = {FileId}")]
+public sealed class HostedFileContent : AIContent
+{
+    private string _fileId;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="HostedFileContent"/> class.
+    /// </summary>
+    /// <param name="fileId">The ID of the hosted file.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="fileId"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentException"><paramref name="fileId"/> is empty or composed entirely of whitespace.</exception>
+    public HostedFileContent(string fileId)
+    {
+        _fileId = Throw.IfNullOrWhitespace(fileId);
+    }
+
+    /// <summary>
+    /// Gets or sets the ID of the hosted file.
+    /// </summary>
+    /// <exception cref="ArgumentNullException"><paramref name="value"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentException"><paramref name="value"/> is empty or composed entirely of whitespace.</exception>
+    public string FileId
+    {
+        get => _fileId;
+        set => _fileId = Throw.IfNullOrWhitespace(value);
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedVectorStoreContent.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a vector store that is hosted by the AI service.
+/// </summary>
+/// <remarks>
+/// Unlike <see cref="HostedFileContent"/> which represents a specific file that is hosted by the AI service,
+/// <see cref="HostedVectorStoreContent"/> represents a vector store that can contain multiple files, indexed
+/// for searching.
+/// </remarks>
+[DebuggerDisplay("VectorStoreId = {VectorStoreId}")]
+public sealed class HostedVectorStoreContent : AIContent
+{
+    private string _vectorStoreId;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="HostedVectorStoreContent"/> class.
+    /// </summary>
+    /// <param name="vectorStoreId">The ID of the hosted file store.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="vectorStoreId"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentException"><paramref name="vectorStoreId"/> is empty or composed entirely of whitespace.</exception>
+    public HostedVectorStoreContent(string vectorStoreId)
+    {
+        _vectorStoreId = Throw.IfNullOrWhitespace(vectorStoreId);
+    }
+
+    /// <summary>
+    /// Gets or sets the ID of the hosted vector store.
+    /// </summary>
+    /// <exception cref="ArgumentNullException"><paramref name="value"/> is <see langword="null"/>.</exception>
+    /// <exception cref="ArgumentException"><paramref name="value"/> is empty or composed entirely of whitespace.</exception>
+    public string VectorStoreId
+    {
+        get => _vectorStoreId;
+        set => _vectorStoreId = Throw.IfNullOrWhitespace(value);
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/HostedCodeInterpreterTool.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Generic;
+
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents a hosted tool that can be specified to an AI service to enable it to execute code it generates.</summary>
@@ -14,4 +16,12 @@ public class HostedCodeInterpreterTool : AITool
     public HostedCodeInterpreterTool()
     {
     }
+
+    /// <summary>Gets or sets a collection of <see cref="AIContent"/> to be used as input to the code interpreter tool.</summary>
+    /// <remarks>
+    /// Services support different varied kinds of inputs. Most support the IDs of files that are hosted by the service,
+    /// represented via <see cref="HostedFileContent"/>. Some also support binary data, represented via <see cref="DataContent"/>.
+    /// Unsupported inputs will be ignored by the <see cref="IChatClient"/> to which the tool is passed.
+    /// </remarks>
+    public IList<AIContent>? Inputs { get; set; }
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/HostedFileSearchTool.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents a hosted tool that can be specified to an AI service to enable it to perform file search operations.</summary>
+/// <remarks>
+/// This tool is designed to facilitate file search functionality within AI services. It allows the service to search
+/// for relevant content based on the provided inputs and constraints, such as the maximum number of results.
+/// </remarks>
+public class HostedFileSearchTool : AITool
+{
+    /// <summary>Initializes a new instance of the <see cref="HostedFileSearchTool"/> class.</summary>
+    public HostedFileSearchTool()
+    {
+    }
+
+    /// <summary>Gets or sets a collection of <see cref="AIContent"/> to be used as input to the file search tool.</summary>
+    /// <remarks>
+    /// If no explicit inputs are provided, the service will determine what inputs should be searched. Different services
+    /// support different kinds of inputs, e.g. some may respect <see cref="HostedFileContent"/> using provider-specific file IDs,
+    /// others may support binary data uploaded as part of the request in <see cref="DataContent"/>, while others may support
+    /// content in a hosted vector store and represented by a <see cref="HostedVectorStoreContent"/>.
+    /// </remarks>
+    public IList<AIContent>? Inputs { get; set; }
+
+    /// <summary>Gets or sets a requested bound on the number of matches the tool should produce.</summary>
+    public int? MaximumResultCount { get; set; }
+}