Add MEAI.Evaluation libraries

Author: shyamnamboodiripad

eng/Version.Details.xml

@@ -1,5 +1,9 @@
 <Dependencies>
   <ProductDependencies>
+    <Dependency Name="Microsoft.Bcl.Memory" Version="9.0.1" CoherentParentDependency="Microsoft.AspNetCore.App.Runtime.win-x64">
+      <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
+      <Sha>69ae1ac0682f4b69b1d733fbf3bdee2825a2aab2</Sha>
+    </Dependency>
     <Dependency Name="Microsoft.Bcl.TimeProvider" Version="9.0.1" CoherentParentDependency="Microsoft.AspNetCore.App.Runtime.win-x64">
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>69ae1ac0682f4b69b1d733fbf3bdee2825a2aab2</Sha>

eng/Versions.props

@@ -28,6 +28,7 @@
   <PropertyGroup Label="Automated">
     <!-- Packages from dotnet/runtime -->
     <MicrosoftBclAsyncInterfacesVersion>9.0.1</MicrosoftBclAsyncInterfacesVersion>
+    <MicrosoftBclMemoryVersion>9.0.1</MicrosoftBclMemoryVersion>
     <MicrosoftBclTimeProviderVersion>9.0.1</MicrosoftBclTimeProviderVersion>
     <MicrosoftExtensionsCachingAbstractionsVersion>9.0.1</MicrosoftExtensionsCachingAbstractionsVersion>
     <MicrosoftExtensionsCachingMemoryVersion>9.0.1</MicrosoftExtensionsCachingMemoryVersion>
@@ -78,6 +79,7 @@
   <PropertyGroup Label="Manual">
     <!-- Packages from dotnet/runtime -->
     <MicrosoftBclAsyncInterfacesLTSVersion>8.0.0</MicrosoftBclAsyncInterfacesLTSVersion>
+    <MicrosoftBclMemoryLTSVersion>9.0.1</MicrosoftBclMemoryLTSVersion>
     <MicrosoftBclTimeProviderLTSVersion>8.0.1</MicrosoftBclTimeProviderLTSVersion>
     <MicrosoftExtensionsCachingAbstractionsLTSVersion>8.0.0</MicrosoftExtensionsCachingAbstractionsLTSVersion>
     <MicrosoftExtensionsCachingMemoryLTSVersion>8.0.1</MicrosoftExtensionsCachingMemoryLTSVersion>
@@ -137,6 +139,7 @@
       Win-x64 is used here because we have picked an arbitrary runtime identifier to flow the version of the latest NETCore.App runtime.
       All Runtime.$rid packages should have the same version.
     -->
+    <MicrosoftMLTokenizersVersion>1.0.1</MicrosoftMLTokenizersVersion>
     <MicrosoftNETCoreAppRuntimeVersion>$(MicrosoftNETCoreAppRuntimewinx64Version)</MicrosoftNETCoreAppRuntimeVersion>
     <!-- Compatibility with VS 17.8/.NET SDK 8.0.1xx -->
     <MicrosoftCodeAnalysisVersion>4.8.0</MicrosoftCodeAnalysisVersion>

eng/packages/General-LTS.props

@@ -4,6 +4,7 @@
   of the framework, we should use the following LTS versions instead -->
   <ItemGroup Condition="'$(TargetFramework)' != 'net9.0' And '$(ForceLatestDotnetVersions)' != 'true'">
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="$(MicrosoftBclAsyncInterfacesLTSVersion)" />
+    <PackageVersion Include="Microsoft.Bcl.Memory" Version="$(MicrosoftBclMemoryLTSVersion)" />
     <PackageVersion Include="Microsoft.Bcl.TimeProvider" Version="$(MicrosoftBclTimeProviderLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="$(MicrosoftExtensionsCachingAbstractionsLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="$(MicrosoftExtensionsCachingMemoryLTSVersion)" />
@@ -23,6 +24,7 @@
     <PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="$(MicrosoftExtensionsLoggingConfigurationLTSVersion)" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="$(MicrosoftExtensionsObjectPoolLTSVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsOptionsConfigurationExtensionsLTSVersion)" />

eng/packages/General-net9.props

@@ -4,6 +4,7 @@
   of the framework, the following versions should be used. -->
   <ItemGroup Condition="'$(TargetFramework)' == 'net9.0' Or '$(ForceLatestDotnetVersions)' == 'true'">
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="$(MicrosoftBclAsyncInterfacesVersion)" />
+    <PackageVersion Include="Microsoft.Bcl.Memory" Version="$(MicrosoftBclMemoryLTSVersion)" />
     <PackageVersion Include="Microsoft.Bcl.TimeProvider" Version="$(MicrosoftBclTimeProviderVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Abstractions" Version="$(MicrosoftExtensionsCachingAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="$(MicrosoftExtensionsCachingMemoryVersion)" />
@@ -23,6 +24,7 @@
     <PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="$(MicrosoftExtensionsLoggingConfigurationVersion)" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
     <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="$(MicrosoftExtensionsObjectPoolVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsOptionsConfigurationExtensionsVersion)" />

eng/packages/General.props

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
+    <PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.21.0" />
     <PackageVersion Include="Azure.AI.Inference" Version="1.0.0-beta.2" />
     <PackageVersion Include="ICSharpCode.Decompiler" Version="8.2.0.7535" />
     <PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
@@ -10,6 +11,7 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisVersion)" />
     <PackageVersion Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" />
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="OpenAI" Version="2.2.0-beta.1" />
     <PackageVersion Include="Polly" Version="8.4.2" />

eng/packages/TestOnly.props

@@ -3,6 +3,7 @@
   <ItemGroup>
     <PackageVersion Include="AutoFixture.AutoMoq" Version="4.17.0" />
     <PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.1" />
+    <PackageVersion Include="Azure.Identity" Version="1.13.2" />
     <PackageVersion Include="autofixture" Version="4.17.0" />
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.5" />
     <PackageVersion Include="FluentAssertions" Version="6.11.0" />
@@ -12,7 +13,7 @@
     <PackageVersion Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.1.3" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
-    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Moq.AutoMock" Version="3.1.0" />
     <PackageVersion Include="Moq" Version="4.18.4" />
     <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.9.0" />

eng/pipelines/templates/BuildAndTest.yml

@@ -23,6 +23,12 @@ parameters:
     default: false
 
 steps:
+  - task: NodeTool@0
+    displayName: Add NodeJS/npm
+    inputs:
+      versionSpec: "20.x"
+      checkLatest: true
+
   - script: ${{ parameters.buildScript }}
             -restore
             /bl:${{ parameters.repoLogPath }}/restore.binlog

eng/xunit.runner.json

@@ -1,4 +1,5 @@
 {
   "diagnosticMessages": true,
-  "longRunningTestSeconds": 300
+  "longRunningTestSeconds": 300,
+  "shadowCopy": false
 }

scripts/ConfigureEvaluationTests.ps1

@@ -0,0 +1,45 @@
+param (
+    [switch]$Configure=$False,
+    [switch]$Unconfigure=$False,
+    [string]$ConfigRoot=$Null
+)
+
+Write-Host "$PSScriptRoot"
+
+if ($Configure -and $Unconfigure) {
+    Write-Error -Message "Cannot specify both -Configure and -Unconfigure"
+    Exit 1
+}
+
+if ($ConfigRoot -eq $Null) {
+    $ConfigRoot = "$HOME/.config/dotnet-extensions"
+}
+
+$ProjectRoot = Resolve-Path "$PSScriptRoot/../test/Libraries"
+$ReportingConfig = "Microsoft.Extensions.AI.Evaluation.Reporting.Tests/appsettings.local.json"
+$IntegrationConfig = "Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.local.json"
+
+if ($Configure) {
+    if (!(Test-Path -Path "$ConfigRoot/$ReportingConfig")) {
+        Write-Host "No configuration found at $ConfigRoot/$ReportingConfig"
+        Exit 0
+    }
+    if (!(Test-Path -Path "$ConfigRoot/$IntegrationConfig")) {
+        Write-Host "No configuration found at $ConfigRoot/$IntegrationConfig"
+        Exit 0
+    }
+
+    Copy-Item -Path "$ConfigRoot/$ReportingConfig" -Destination "$ProjectRoot/$ReportingConfig" -Force
+    Copy-Item -Path "$ConfigRoot/$IntegrationConfig" -Destination "$ProjectRoot/$IntegrationConfig" -Force
+
+    Write-Host "Test configured to use external resources"
+} elseif ($Unconfigure) {
+    Remove-Item -Path "$ProjectRoot/$ReportingConfig" -Force
+    Remove-Item -Path "$ProjectRoot/$IntegrationConfig" -Force
+
+    Write-Host "Test unconfigured from using external resources"
+} else {
+    Write-Error -Message "Must specify either -Configure or -Unconfigure"
+    Exit 1
+}
+

src/Libraries/Directory.Build.props

@@ -11,7 +11,6 @@
     <InjectTrimAttributesOnLegacy Condition="'$(InjectTrimAttributesOnLegacy)' == ''">true</InjectTrimAttributesOnLegacy>
     <InjectSkipLocalsInitAttributeOnLegacy Condition="'$(InjectSkipLocalsInitAttributeOnLegacy)' == ''">true</InjectSkipLocalsInitAttributeOnLegacy>
     <InjectCallerAttributesOnLegacy Condition="'$(InjectCallerAttributesOnLegacy)' == ''">true</InjectCallerAttributesOnLegacy>
-    <InjectDiagnosticAttributesOnLegacy Condition="'$(InjectDiagnosticAttributesOnLegacy)' == ''">true</InjectDiagnosticAttributesOnLegacy>
     <InjectSharedThrow Condition="'$(InjectSharedThrow)' == ''">true</InjectSharedThrow>
   </PropertyGroup>
 </Project>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
+using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
+
+internal sealed class CleanCacheCommand(ILogger logger)
+{
+    internal async Task<int> InvokeAsync(DirectoryInfo storageRootDir, CancellationToken cancellationToken = default)
+    {
+        string storageRootPath = storageRootDir.FullName;
+        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+        logger.LogInformation("Deleting expired cache entries...");
+
+        var cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
+
+        await logger.ExecuteWithCatchAsync(
+            () => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);
+
+        return 0;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs

@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
+using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
+
+internal sealed class CleanResultsCommand(ILogger logger)
+{
+    internal async Task<int> InvokeAsync(
+        DirectoryInfo storageRootDir,
+        int lastN,
+        CancellationToken cancellationToken = default)
+    {
+        string storageRootPath = storageRootDir.FullName;
+        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+
+        var resultStore = new DiskBasedResultStore(storageRootPath);
+
+        await logger.ExecuteWithCatchAsync(
+            async ValueTask () =>
+            {
+                if (lastN is 0)
+                {
+                    logger.LogInformation("Deleting all results...");
+
+                    await resultStore.DeleteResultsAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    logger.LogInformation("Deleting all results except the {lastN} most recent ones...", lastN);
+
+                    HashSet<string> toPreserve = [];
+
+                    await foreach (string executionName in
+                        resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
+                    {
+                        cancellationToken.ThrowIfCancellationRequested();
+
+                        _ = toPreserve.Add(executionName);
+                    }
+
+                    await foreach (string executionName in
+                        resultStore.GetLatestExecutionNamesAsync(
+                            cancellationToken: cancellationToken).ConfigureAwait(false))
+                    {
+                        cancellationToken.ThrowIfCancellationRequested();
+
+                        if (!toPreserve.Contains(executionName))
+                        {
+                            await resultStore.DeleteResultsAsync(
+                                executionName,
+                                cancellationToken: cancellationToken).ConfigureAwait(false);
+                        }
+                    }
+                }
+            }).ConfigureAwait(false);
+
+        return 0;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.Format.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
+
+internal partial class ReportCommand
+{
+    internal enum Format
+    {
+        html,
+        json
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs

@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI.Evaluation.Reporting;
+using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Html;
+using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Json;
+using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
+
+internal sealed partial class ReportCommand(ILogger logger)
+{
+    internal async Task<int> InvokeAsync(
+        DirectoryInfo storageRootDir,
+        FileInfo outputFile,
+        int lastN,
+        Format format,
+        CancellationToken cancellationToken = default)
+    {
+        string storageRootPath = storageRootDir.FullName;
+        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+
+        var results = new List<ScenarioRunResult>();
+        var resultStore = new DiskBasedResultStore(storageRootPath);
+
+        await foreach (string executionName in
+            resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            await foreach (ScenarioRunResult result in
+                resultStore.ReadResultsAsync(
+                    executionName,
+                    cancellationToken: cancellationToken).ConfigureAwait(false))
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                results.Add(result);
+            }
+        }
+
+        string outputFilePath = outputFile.FullName;
+        string? outputPath = Path.GetDirectoryName(outputFilePath);
+        if (outputPath is not null && !Directory.Exists(outputPath))
+        {
+            _ = Directory.CreateDirectory(outputPath);
+        }
+
+        IEvaluationReportWriter reportWriter = format switch
+        {
+            Format.html => new HtmlReportWriter(outputFilePath),
+            Format.json => new JsonReportWriter(outputFilePath),
+            _ => throw new NotSupportedException(),
+        };
+
+        await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false);
+        logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format);
+
+        return 0;
+    }
+}