Skip to content

Commit 625ed7b

Browse files
stephentoubstephentoub
committed
Merged PR 48946: Backport recent M.E.AI changes from main
Several git cherry-picks with no conflicts or manual changes. ---- #### AI description (iteration 1) #### PR Classification Backport recent changes from the main branch to the current branch. #### PR Summary This pull request backports recent changes related to AI functionalities and tests from the main branch. - `OpenAIChatClientTests.cs`: Added new tests for image content handling in chat messages. - `ChatResponseExtensions.cs`: Enhanced content coalescing to handle multiple content types. - `TextReasoningContent.cs` and `TextReasoningContentTests.cs`: Added new class and corresponding tests for text reasoning content. - `AIFunctionFactory.cs`: Improved result marshalling logic for async methods. - `DistributedCachingChatClient.cs`: Updated cache key computation to include messages and options. <!-- GitOpsUserAgent=GitOps.Apps.Server.pullrequestcopilot -->
2 parents 7e7b3ee + ae724c1 commit 625ed7b

34 files changed

+589
-181
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs‎

Lines changed: 45 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -180,53 +180,60 @@ static async Task<ChatResponse> ToChatResponseAsync(
180180
}
181181
}
182182

183-
/// <summary>Coalesces sequential <see cref="TextContent"/> content elements.</summary>
183+
/// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
184184
internal static void CoalesceTextContent(List<AIContent> contents)
185185
{
186-
StringBuilder? coalescedText = null;
186+
Coalesce<TextContent>(contents, static text => new(text));
187+
Coalesce<TextReasoningContent>(contents, static text => new(text));
187188

188-
// Iterate through all of the items in the list looking for contiguous items that can be coalesced.
189-
int start = 0;
190-
while (start < contents.Count - 1)
189+
// This implementation relies on TContent's ToString returning its exact text.
190+
static void Coalesce<TContent>(List<AIContent> contents, Func<string, TContent> fromText)
191+
where TContent : AIContent
191192
{
192-
// We need at least two TextContents in a row to be able to coalesce.
193-
if (contents[start] is not TextContent firstText)
194-
{
195-
start++;
196-
continue;
197-
}
198-
199-
if (contents[start + 1] is not TextContent secondText)
200-
{
201-
start += 2;
202-
continue;
203-
}
193+
StringBuilder? coalescedText = null;
204194

205-
// Append the text from those nodes and continue appending subsequent TextContents until we run out.
206-
// We null out nodes as their text is appended so that we can later remove them all in one O(N) operation.
207-
coalescedText ??= new();
208-
_ = coalescedText.Clear().Append(firstText.Text).Append(secondText.Text);
209-
contents[start + 1] = null!;
210-
int i = start + 2;
211-
for (; i < contents.Count && contents[i] is TextContent next; i++)
195+
// Iterate through all of the items in the list looking for contiguous items that can be coalesced.
196+
int start = 0;
197+
while (start < contents.Count - 1)
212198
{
213-
_ = coalescedText.Append(next.Text);
214-
contents[i] = null!;
199+
// We need at least two TextContents in a row to be able to coalesce.
200+
if (contents[start] is not TContent firstText)
201+
{
202+
start++;
203+
continue;
204+
}
205+
206+
if (contents[start + 1] is not TContent secondText)
207+
{
208+
start += 2;
209+
continue;
210+
}
211+
212+
// Append the text from those nodes and continue appending subsequent TextContents until we run out.
213+
// We null out nodes as their text is appended so that we can later remove them all in one O(N) operation.
214+
coalescedText ??= new();
215+
_ = coalescedText.Clear().Append(firstText).Append(secondText);
216+
contents[start + 1] = null!;
217+
int i = start + 2;
218+
for (; i < contents.Count && contents[i] is TContent next; i++)
219+
{
220+
_ = coalescedText.Append(next);
221+
contents[i] = null!;
222+
}
223+
224+
// Store the replacement node. We inherit the properties of the first text node. We don't
225+
// currently propagate additional properties from the subsequent nodes. If we ever need to,
226+
// we can add that here.
227+
var newContent = fromText(coalescedText.ToString());
228+
contents[start] = newContent;
229+
newContent.AdditionalProperties = firstText.AdditionalProperties?.Clone();
230+
231+
start = i;
215232
}
216233

217-
// Store the replacement node.
218-
contents[start] = new TextContent(coalescedText.ToString())
219-
{
220-
// We inherit the properties of the first text node. We don't currently propagate additional
221-
// properties from the subsequent nodes. If we ever need to, we can add that here.
222-
AdditionalProperties = firstText.AdditionalProperties?.Clone(),
223-
};
224-
225-
start = i;
234+
// Remove all of the null slots left over from the coalescing process.
235+
_ = contents.RemoveAll(u => u is null);
226236
}
227-
228-
// Remove all of the null slots left over from the coalescing process.
229-
_ = contents.RemoveAll(u => u is null);
230237
}
231238

232239
/// <summary>Finalizes the <paramref name="response"/> object.</summary>

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace Microsoft.Extensions.AI;
1212
[JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: "functionCall")]
1313
[JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: "functionResult")]
1414
[JsonDerivedType(typeof(TextContent), typeDiscriminator: "text")]
15+
[JsonDerivedType(typeof(TextReasoningContent), typeDiscriminator: "reasoning")]
1516
[JsonDerivedType(typeof(UriContent), typeDiscriminator: "uri")]
1617
[JsonDerivedType(typeof(UsageContent), typeDiscriminator: "usage")]
1718
public class AIContent

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ErrorContent.cs‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33

44
using System.Diagnostics;
5-
using System.Text.Json.Serialization;
6-
using Microsoft.Shared.Diagnostics;
5+
using System.Diagnostics.CodeAnalysis;
76

87
namespace Microsoft.Extensions.AI;
98

@@ -16,33 +15,33 @@ namespace Microsoft.Extensions.AI;
1615
public class ErrorContent : AIContent
1716
{
1817
/// <summary>The error message.</summary>
19-
private string _message;
18+
private string? _message;
2019

21-
/// <summary>Initializes a new instance of the <see cref="ErrorContent"/> class with the specified message.</summary>
22-
/// <param name="message">The message to store in this content.</param>
23-
[JsonConstructor]
24-
public ErrorContent(string message)
20+
/// <summary>Initializes a new instance of the <see cref="ErrorContent"/> class with the specified error message.</summary>
21+
/// <param name="message">The error message to store in this content.</param>
22+
public ErrorContent(string? message)
2523
{
26-
_message = Throw.IfNull(message);
24+
_message = message;
2725
}
2826

2927
/// <summary>Gets or sets the error message.</summary>
28+
[AllowNull]
3029
public string Message
3130
{
32-
get => _message;
33-
set => _message = Throw.IfNull(value);
31+
get => _message ?? string.Empty;
32+
set => _message = value;
3433
}
3534

36-
/// <summary>Gets or sets the error code.</summary>
35+
/// <summary>Gets or sets an error code associated with the error.</summary>
3736
public string? ErrorCode { get; set; }
3837

39-
/// <summary>Gets or sets the error details.</summary>
38+
/// <summary>Gets or sets additional details about the error.</summary>
4039
public string? Details { get; set; }
4140

4241
/// <summary>Gets a string representing this instance to display in the debugger.</summary>
4342
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
4443
private string DebuggerDisplay =>
45-
$"Error = {Message}" +
46-
(ErrorCode is not null ? $" ({ErrorCode})" : string.Empty) +
47-
(Details is not null ? $" - {Details}" : string.Empty);
44+
$"Error = \"{Message}\"" +
45+
(!string.IsNullOrWhiteSpace(ErrorCode) ? $" ({ErrorCode})" : string.Empty) +
46+
(!string.IsNullOrWhiteSpace(Details) ? $" - \"{Details}\"" : string.Empty);
4847
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextReasoningContent.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Diagnostics;
5+
using System.Diagnostics.CodeAnalysis;
6+
7+
namespace Microsoft.Extensions.AI;
8+
9+
/// <summary>
10+
/// Represents text reasoning content in a chat.
11+
/// </summary>
12+
/// <remarks>
13+
/// <see cref="TextReasoningContent"/> is distinct from <see cref="TextContent"/>. <see cref="TextReasoningContent"/>
14+
/// represents "thinking" or "reasoning" performed by the model and is distinct from the actual output text from
15+
/// the model, which is represented by <see cref="TextContent"/>. Neither types derives from the other.
16+
/// </remarks>
17+
[DebuggerDisplay("{DebuggerDisplay,nq}")]
18+
public sealed class TextReasoningContent : AIContent
19+
{
20+
private string? _text;
21+
22+
/// <summary>
23+
/// Initializes a new instance of the <see cref="TextReasoningContent"/> class.
24+
/// </summary>
25+
/// <param name="text">The text reasoning content.</param>
26+
public TextReasoningContent(string? text)
27+
{
28+
_text = text;
29+
}
30+
31+
/// <summary>
32+
/// Gets or sets the text reasoning content.
33+
/// </summary>
34+
[AllowNull]
35+
public string Text
36+
{
37+
get => _text ?? string.Empty;
38+
set => _text = value;
39+
}
40+
41+
/// <inheritdoc/>
42+
public override string ToString() => Text;
43+
44+
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
45+
private string DebuggerDisplay => $"Reasoning = \"{Text}\"";
46+
}

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -42,30 +42,18 @@ public static partial class AIJsonUtilities
4242
[UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", Justification = "DefaultJsonTypeInfoResolver is only used when reflection-based serialization is enabled")]
4343
private static JsonSerializerOptions CreateDefaultOptions()
4444
{
45-
// If reflection-based serialization is enabled by default, use it, as it's the most permissive in terms of what it can serialize,
46-
// and we want to be flexible in terms of what can be put into the various collections in the object model.
47-
// Otherwise, use the source-generated options to enable trimming and Native AOT.
48-
JsonSerializerOptions options;
45+
// Copy configuration from the source generated context.
46+
JsonSerializerOptions options = new(JsonContext.Default.Options)
47+
{
48+
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
49+
};
4950

5051
if (JsonSerializer.IsReflectionEnabledByDefault)
5152
{
52-
// Keep in sync with the JsonSourceGenerationOptions attribute on JsonContext below.
53-
options = new(JsonSerializerDefaults.Web)
54-
{
55-
TypeInfoResolver = new DefaultJsonTypeInfoResolver(),
56-
Converters = { new JsonStringEnumConverter() },
57-
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
58-
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
59-
WriteIndented = true,
60-
};
61-
}
62-
else
63-
{
64-
options = new(JsonContext.Default.Options)
65-
{
66-
// Compile-time encoder setting not yet available
67-
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
68-
};
53+
// If reflection-based serialization is enabled by default, use it as a fallback for all other types.
54+
// Also turn on string-based enum serialization for all unknown enums.
55+
options.TypeInfoResolverChain.Add(new DefaultJsonTypeInfoResolver());
56+
options.Converters.Add(new JsonStringEnumConverter());
6957
}
7058

7159
options.MakeReadOnly();
@@ -83,6 +71,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
8371
[JsonSerializable(typeof(SpeechToTextResponseUpdate))]
8472
[JsonSerializable(typeof(IReadOnlyList<SpeechToTextResponseUpdate>))]
8573
[JsonSerializable(typeof(IList<ChatMessage>))]
74+
[JsonSerializable(typeof(IEnumerable<ChatMessage>))]
75+
[JsonSerializable(typeof(ChatMessage[]))]
8676
[JsonSerializable(typeof(ChatOptions))]
8777
[JsonSerializable(typeof(EmbeddingGenerationOptions))]
8878
[JsonSerializable(typeof(ChatClientMetadata))]
@@ -95,14 +85,24 @@ private static JsonSerializerOptions CreateDefaultOptions()
9585
[JsonSerializable(typeof(JsonDocument))]
9686
[JsonSerializable(typeof(JsonElement))]
9787
[JsonSerializable(typeof(JsonNode))]
88+
[JsonSerializable(typeof(JsonObject))]
89+
[JsonSerializable(typeof(JsonValue))]
90+
[JsonSerializable(typeof(JsonArray))]
9891
[JsonSerializable(typeof(IEnumerable<string>))]
92+
[JsonSerializable(typeof(char))]
9993
[JsonSerializable(typeof(string))]
10094
[JsonSerializable(typeof(int))]
95+
[JsonSerializable(typeof(short))]
10196
[JsonSerializable(typeof(long))]
97+
[JsonSerializable(typeof(uint))]
98+
[JsonSerializable(typeof(ushort))]
99+
[JsonSerializable(typeof(ulong))]
102100
[JsonSerializable(typeof(float))]
103101
[JsonSerializable(typeof(double))]
102+
[JsonSerializable(typeof(decimal))]
104103
[JsonSerializable(typeof(bool))]
105104
[JsonSerializable(typeof(TimeSpan))]
105+
[JsonSerializable(typeof(DateTime))]
106106
[JsonSerializable(typeof(DateTimeOffset))]
107107
[JsonSerializable(typeof(Embedding))]
108108
[JsonSerializable(typeof(Embedding<byte>))]

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
9090

9191
// For cases where the hash may be used as a cache key, we rely on collision resistance for security purposes.
9292
// If a collision occurs, we'd serve a cached LLM response for a potentially unrelated prompt, leading to information
93-
// disclosure. Use of SHA256 is an implementation detail and can be easily swapped in the future if needed, albeit
93+
// disclosure. Use of SHA384 is an implementation detail and can be easily swapped in the future if needed, albeit
9494
// invalidating any existing cache entries.
9595
#if NET
9696
IncrementalHashStream? stream = IncrementalHashStream.ThreadStaticInstance;
@@ -107,7 +107,7 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
107107
stream = new();
108108
}
109109

110-
Span<byte> hashData = stackalloc byte[SHA256.HashSizeInBytes];
110+
Span<byte> hashData = stackalloc byte[SHA384.HashSizeInBytes];
111111
try
112112
{
113113
foreach (object? value in values)
@@ -133,8 +133,8 @@ public static string HashDataToString(ReadOnlySpan<object?> values, JsonSerializ
133133
JsonSerializer.Serialize(stream, value, jti);
134134
}
135135

136-
using var sha256 = SHA256.Create();
137-
var hashData = sha256.ComputeHash(stream.GetBuffer(), 0, (int)stream.Length);
136+
using var hashAlgorithm = SHA384.Create();
137+
var hashData = hashAlgorithm.ComputeHash(stream.GetBuffer(), 0, (int)stream.Length);
138138

139139
return ConvertToHexString(hashData);
140140

@@ -185,7 +185,7 @@ private sealed class IncrementalHashStream : Stream
185185
public static IncrementalHashStream? ThreadStaticInstance;
186186

187187
/// <summary>The <see cref="IncrementalHash"/> used by this instance.</summary>
188-
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
188+
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA384);
189189

190190
/// <summary>Gets the current hash and resets.</summary>
191191
public void GetHashAndReset(Span<byte> bytes) => _hash.GetHashAndReset(bytes);

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,6 @@ protected override async Task WriteCacheStreamingAsync(
125125
}
126126
}
127127

128-
protected override string GetCacheKey(params ReadOnlySpan<object?> values)
129-
=> base.GetCacheKey([.. values, .. _cachingKeys]);
128+
protected override string GetCacheKey(IEnumerable<ChatMessage> messages, ChatOptions? options, params ReadOnlySpan<object?> additionalValues)
129+
=> base.GetCacheKey(messages, options, [.. additionalValues, .. _cachingKeys]);
130130
}

‎src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,11 +190,11 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
190190
break;
191191

192192
case UriContent uriContent when uriContent.HasTopLevelMediaType("image"):
193-
parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri));
193+
parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)));
194194
break;
195195

196196
case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
197-
parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
197+
parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)));
198198
break;
199199

200200
case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"):
@@ -220,6 +220,21 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
220220
return parts;
221221
}
222222

223+
private static ChatImageDetailLevel? GetImageDetail(AIContent content)
224+
{
225+
if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true)
226+
{
227+
return value switch
228+
{
229+
string detailString => new ChatImageDetailLevel(detailString),
230+
ChatImageDetailLevel detail => detail,
231+
_ => null
232+
};
233+
}
234+
235+
return null;
236+
}
237+
223238
private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingChatCompletionAsync(
224239
IAsyncEnumerable<StreamingChatCompletionUpdate> updates,
225240
[EnumeratorCancellation] CancellationToken cancellationToken = default)

0 commit comments

Comments
 (0)