Introduce IntentResolutionEvaluator

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/IntentResolutionEvaluator.cs

@@ -0,0 +1,88 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality;
+
+/// <summary>
+/// Contextual information that the <see cref="IntentResolutionEvaluator"/> uses to evaluate an AI system's
+/// effectiveness at identifying and resolving user intent.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <see cref="IntentResolutionEvaluator"/> evaluates an AI system's effectiveness at identifying and resolving user
+/// intent based on the supplied conversation history and the tool definitions supplied via
+/// <see cref="ToolDefinitions"/>.
+/// </para>
+/// <para>
+/// Note that at the moment, <see cref="IntentResolutionEvaluator"/> only supports evaluating calls to tools that are
+/// defined as <see cref="AIFunction"/>s. Any other <see cref="AITool"/> definitions that are supplied via
+/// <see cref="ToolDefinitions"/> will be ignored.
+/// </para>
+/// </remarks>
+[Experimental("AIEVAL001")]
+public sealed class IntentResolutionEvaluatorContext : EvaluationContext
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="IntentResolutionEvaluatorContext"/> class.
+    /// </summary>
+    /// <param name="toolDefinitions">
+    /// <para>
+    /// The set of tool definitions (see <see cref="ChatOptions.Tools"/>) that were used when generating the model
+    /// response that is being evaluated.
+    /// </para>
+    /// <para>
+    /// Note that at the moment, <see cref="IntentResolutionEvaluator"/> only supports evaluating calls to tools that
+    /// are defined as <see cref="AIFunction"/>s. Any other <see cref="AITool"/> definitions will be ignored.
+    /// </para>
+    /// </param>
+    public IntentResolutionEvaluatorContext(IEnumerable<AITool> toolDefinitions)
+        : base(name: IntentResolutionContextName, contents: [new TextContent(toolDefinitions.RenderAsJson())])
+    {
+        ToolDefinitions = [.. toolDefinitions];
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="IntentResolutionEvaluatorContext"/> class.
+    /// </summary>
+    /// <param name="toolDefinitions">
+    /// <para>
+    /// The set of tool definitions (see <see cref="ChatOptions.Tools"/>) that were used when generating the model
+    /// response that is being evaluated.
+    /// </para>
+    /// <para>
+    /// Note that at the moment, <see cref="IntentResolutionEvaluator"/> only supports evaluating calls to tools that
+    /// are defined as <see cref="AIFunction"/>s. Any other <see cref="AITool"/> definitions will be ignored.
+    /// </para>
+    /// </param>
+    public IntentResolutionEvaluatorContext(params AITool[] toolDefinitions)
+        : this(toolDefinitions as IEnumerable<AITool>)
+    {
+    }
+
+    /// <summary>
+    /// Gets the unique <see cref="EvaluationContext.Name"/> that is used for
+    /// <see cref="IntentResolutionEvaluatorContext"/>.
+    /// </summary>
+    public static string IntentResolutionContextName => "Tool Definitions (Intent Resolution)";
+
+    /// <summary>
+    /// Gets set of tool definitions (see <see cref="ChatOptions.Tools"/>) that were used when generating the model
+    /// response that is being evaluated.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// <see cref="IntentResolutionEvaluator"/> evaluates an AI system's effectiveness at identifying and resolving user
+    /// intent based on the supplied conversation history and the tool definitions supplied via
+    /// <see cref="ToolDefinitions"/>.
+    /// </para>
+    /// <para>
+    /// Note that at the moment, <see cref="IntentResolutionEvaluator"/> only supports evaluating calls to tools that
+    /// are defined as <see cref="AIFunction"/>s. Any other <see cref="AITool"/> definitions that are supplied via
+    /// <see cref="ToolDefinitions"/> will be ignored.
+    /// </para>
+    /// </remarks>
+    public IReadOnlyList<AITool> ToolDefinitions { get; }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/IntentResolutionEvaluatorContext.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.AI.Evaluation.Quality.JsonSerialization;
+using Microsoft.Extensions.AI.Evaluation.Quality.Utilities;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality;
+
+internal sealed class IntentResolutionRating
+{
+    public static IntentResolutionRating Inconclusive { get; } = new IntentResolutionRating();
+
+    [JsonPropertyName("resolution_score")]
+    public int ResolutionScore { get; }
+
+    [JsonPropertyName("explanation")]
+    public string? Explanation { get; }
+
+    [JsonPropertyName("agent_perceived_intent")]
+    public string? AgentPerceivedIntent { get; }
+
+    [JsonPropertyName("actual_user_intent")]
+    public string? ActualUserIntent { get; }
+
+    [JsonPropertyName("conversation_has_intent")]
+    public bool ConversationHasIntent { get; }
+
+    [JsonPropertyName("correct_intent_detected")]
+    public bool CorrectIntentDetected { get; }
+
+    [JsonPropertyName("intent_resolved")]
+    public bool IntentResolved { get; }
+
+    private const int MinValue = 1;
+    private const int MaxValue = 5;
+
+    public bool IsInconclusive => ResolutionScore < MinValue || ResolutionScore > MaxValue;
+
+    [JsonConstructor]
+#pragma warning disable S107 // Methods should not have too many parameters
+    public IntentResolutionRating(
+        int resolutionScore = -1,
+        string? explanation = null,
+        string? agentPerceivedIntent = null,
+        string? actualUserIntent = null,
+        bool conversationHasIntent = false,
+        bool correctIntentDetected = false,
+        bool intentResolved = false)
+#pragma warning restore S107
+    {
+        ResolutionScore = resolutionScore;
+        Explanation = explanation;
+        AgentPerceivedIntent = agentPerceivedIntent;
+        ActualUserIntent = actualUserIntent;
+        ConversationHasIntent = conversationHasIntent;
+        CorrectIntentDetected = correctIntentDetected;
+        IntentResolved = intentResolved;
+    }
+
+    public static IntentResolutionRating FromJson(string jsonResponse)
+    {
+        ReadOnlySpan<char> trimmed = JsonOutputFixer.TrimMarkdownDelimiters(jsonResponse);
+        return JsonSerializer.Deserialize(trimmed, SerializerContext.Default.IntentResolutionRating)!;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/IntentResolutionRating.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality.JsonSerialization;
+
+[JsonSourceGenerationOptions(
+    WriteIndented = true,
+    AllowTrailingCommas = true,
+    PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
+[JsonSerializable(typeof(RelevanceTruthAndCompletenessRating))]
+[JsonSerializable(typeof(IntentResolutionRating))]
+internal sealed partial class SerializerContext : JsonSerializerContext;

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/JsonSerialization/SerializerContext.cs

@@ -43,7 +43,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Quality;
 /// Tutorial: Evaluate a model's response with response caching and reporting.
 /// </related>
 [Experimental("AIEVAL001")]
-public sealed partial class RelevanceTruthAndCompletenessEvaluator : IEvaluator
+public sealed class RelevanceTruthAndCompletenessEvaluator : IEvaluator
 {
     /// <summary>
     /// Gets the <see cref="EvaluationMetric.Name"/> of the <see cref="NumericMetric"/> returned by
@@ -271,20 +271,20 @@ private static async ValueTask ParseEvaluationResponseAsync(
         ChatConfiguration chatConfiguration,
         CancellationToken cancellationToken)
     {
-        Rating rating;
+        RelevanceTruthAndCompletenessRating rating;
 
         string evaluationResponseText = evaluationResponse.Text.Trim();
         if (string.IsNullOrEmpty(evaluationResponseText))
         {
-            rating = Rating.Inconclusive;
+            rating = RelevanceTruthAndCompletenessRating.Inconclusive;
             result.AddDiagnosticsToAllMetrics(
                 EvaluationDiagnostic.Error("The model failed to produce a valid evaluation response."));
         }
         else
         {
             try
             {
-                rating = Rating.FromJson(evaluationResponseText);
+                rating = RelevanceTruthAndCompletenessRating.FromJson(evaluationResponseText);
             }
             catch (JsonException)
             {
@@ -298,26 +298,26 @@ await JsonOutputFixer.RepairJsonAsync(
 
                     if (string.IsNullOrWhiteSpace(repairedJson))
                     {
-                        rating = Rating.Inconclusive;
+                        rating = RelevanceTruthAndCompletenessRating.Inconclusive;
                         result.AddDiagnosticsToAllMetrics(
                             EvaluationDiagnostic.Error(
                                 $"""
-                                Failed to repair the following response from the model and parse scores for '{RelevanceMetricName}', '{TruthMetricName}' and '{CompletenessMetricName}'.:
+                                Failed to repair the following response from the model and parse scores for '{RelevanceMetricName}', '{TruthMetricName}' and '{CompletenessMetricName}':
                                 {evaluationResponseText}
                                 """));
                     }
                     else
                     {
-                        rating = Rating.FromJson(repairedJson);
+                        rating = RelevanceTruthAndCompletenessRating.FromJson(repairedJson);
                     }
                 }
                 catch (JsonException ex)
                 {
-                    rating = Rating.Inconclusive;
+                    rating = RelevanceTruthAndCompletenessRating.Inconclusive;
                     result.AddDiagnosticsToAllMetrics(
                         EvaluationDiagnostic.Error(
                             $"""
-                            Failed to repair the following response from the model and parse scores for '{RelevanceMetricName}', '{TruthMetricName}' and '{CompletenessMetricName}'.:
+                            Failed to repair the following response from the model and parse scores for '{RelevanceMetricName}', '{TruthMetricName}' and '{CompletenessMetricName}':
                             {evaluationResponseText}
                             {ex}
                             """));
@@ -336,10 +336,7 @@ void UpdateResult()
             relevance.AddOrUpdateChatMetadata(evaluationResponse, evaluationDuration);
             relevance.Value = rating.Relevance;
             relevance.Interpretation = relevance.InterpretScore();
-            if (!string.IsNullOrWhiteSpace(rating.RelevanceReasoning))
-            {
-                relevance.Reason = rating.RelevanceReasoning!;
-            }
+            relevance.Reason = rating.RelevanceReasoning;
 
             if (rating.RelevanceReasons.Any())
             {
@@ -351,10 +348,7 @@ void UpdateResult()
             truth.AddOrUpdateChatMetadata(evaluationResponse, evaluationDuration);
             truth.Value = rating.Truth;
             truth.Interpretation = truth.InterpretScore();
-            if (!string.IsNullOrWhiteSpace(rating.TruthReasoning))
-            {
-                truth.Reason = rating.TruthReasoning!;
-            }
+            truth.Reason = rating.TruthReasoning;
 
             if (rating.TruthReasons.Any())
             {
@@ -366,21 +360,13 @@ void UpdateResult()
             completeness.AddOrUpdateChatMetadata(evaluationResponse, evaluationDuration);
             completeness.Value = rating.Completeness;
             completeness.Interpretation = completeness.InterpretScore();
-            if (!string.IsNullOrWhiteSpace(rating.CompletenessReasoning))
-            {
-                completeness.Reason = rating.CompletenessReasoning!;
-            }
+            completeness.Reason = rating.CompletenessReasoning;
 
             if (rating.CompletenessReasons.Any())
             {
                 string value = string.Join(Separator, rating.CompletenessReasons);
                 completeness.AddOrUpdateMetadata(name: Rationales, value);
             }
-
-            if (!string.IsNullOrWhiteSpace(rating.Error))
-            {
-                result.AddDiagnosticsToAllMetrics(EvaluationDiagnostic.Error(rating.Error!));
-            }
         }
     }
 }