Adds support for conversion of API manifest document to OpenAI Plugin manifest

.azure-pipelines/ci-build.yml

@@ -84,7 +84,7 @@ stages:
         displayName: 'Run CredScan - Test'
         inputs:
           toolMajorVersion: 'V2'
-          scanFolder: '$(Build.SourcesDirectory)\src\tests'
+          scanFolder: '$(Build.SourcesDirectory)\tests'
           debugMode: false
 
       - task: AntiMalware@3

.github/CODEOWNERS

@@ -1 +1 @@
-* @peombwa @darrelmiller @baywet
+* @peombwa @darrelmiller @baywet @zengin

CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Added support for conversion of API manifest document to OpenAI Plugin manifest. #4
+- Added VerificationTokens property to OpenAI Plugin manifest auth type. #32
+- Added OpenAI Plugin manifest validation. #32
+- Added API Manifest validation. #5
+- Added ApplicationName property to ApiManifestDocument. #5
+
+### Changed
+
+- Renamed Request class to RequestInfo to align with the API manifest specification. #21
+- Renamed Auth property in ApiDependency to AuthorizationRequirements to align with the API manifest specification. #5
+
+## [0.5.1] - 2023-08-17
+
+### Changed
+
+- Fixed typos in properties.
+
+## [0.5.0] - 2023-08-17
+
+### Added
+
+- Initial release of the library.

src/lib/ApiManifestDocument.cs

@@ -17,33 +17,27 @@ public class ApiManifestDocument
 
     public ApiManifestDocument(string applicationName)
     {
-        Validate(applicationName);
-
         ApplicationName = applicationName;
+        Validate();
     }
 
     public ApiManifestDocument(JsonElement value)
     {
         ParsingHelpers.ParseMap(value, this, handlers);
-
-        Validate(ApplicationName);
+        Validate();
     }
 
     // Write method
     public void Write(Utf8JsonWriter writer)
     {
-        Validate(ApplicationName);
-
+        Validate();
         writer.WriteStartObject();
-
         writer.WriteString(ApplicationNameProperty, ApplicationName);
-
         if (Publisher != null)
         {
             writer.WritePropertyName(PublisherProperty);
             Publisher.Write(writer);
         }
-
         if (ApiDependencies.Any())
         {
             writer.WritePropertyName(ApiDependenciesProperty);
@@ -55,13 +49,11 @@ public void Write(Utf8JsonWriter writer)
             }
             writer.WriteEndObject();
         }
-
         if (Extensions.Any())
         {
             writer.WritePropertyName(ExtensionsProperty);
             Extensions.Write(writer);
         }
-
         writer.WriteEndObject();
     }
 
@@ -71,9 +63,10 @@ public static ApiManifestDocument Load(JsonElement value)
         return new ApiManifestDocument(value);
     }
 
-    private static void Validate(string? applicationName)
+    internal void Validate()
     {
-        ValidationHelpers.ValidateNullOrWhitespace(nameof(applicationName), applicationName, nameof(ApiManifestDocument));
+        ValidationHelpers.ValidateNullOrWhitespace(nameof(ApplicationName), ApplicationName, nameof(ApiManifestDocument));
+        Publisher?.Validate();
     }
 
     // Create fixed field map for ApiManifest

src/lib/Constants.cs

@@ -14,5 +14,7 @@ internal static class ErrorMessage
         public static readonly string FieldIsNotValid = "'{0}' is not valid.";
         public static readonly string FieldLengthExceeded = "'{0}' length exceeded. Maximum length allowed is '{1}'.";
         public static readonly string BaseUrlIsNotValid = "The {0} must be a valid URL and end in a slash.";
+        public static readonly string ApiDependencyNotFound = "Failed to get a valid apiDependency from the provided apiManifestDocument. The property is required generate a complete {0}.";
+        public static readonly string ApiDescriptionUrlNotFound = "ApiDescriptionUrl is missing in the provided apiManifestDocument. The property is required generate a complete {0}.";
     }
 }

src/lib/Exceptions/ApiManifestException.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.OpenApi.ApiManifest.Exceptions
+{
+    public class ApiManifestException : Exception
+    {
+        public ApiManifestException(string message) : base(message)
+        {
+        }
+
+        public ApiManifestException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+    }
+}

src/lib/Helpers/ParsingHelpers.cs

@@ -1,13 +1,23 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+using Microsoft.OpenApi.Readers;
 using System.Diagnostics;
+using System.Net;
 using System.Text.Json;
 
 namespace Microsoft.OpenApi.ApiManifest.Helpers;
 
 internal static class ParsingHelpers
 {
+    private static readonly Lazy<HttpClient> s_httpClient = new(() => new HttpClient(new HttpClientHandler()
+    {
+        SslProtocols = System.Security.Authentication.SslProtocols.Tls12,
+    }))
+    {
+        Value = { DefaultRequestVersion = HttpVersion.Version20 }
+    };
+
     internal static void ParseMap<T>(JsonElement node, T permissionsDocument, FixedFieldMap<T> handlers)
     {
         foreach (var element in node.EnumerateObject())
@@ -136,6 +146,38 @@ internal static IEnumerable<KeyValuePair<string, string>> ParseKey(string key)
             yield return keyValue;
         }
     }
+
+    internal static async Task<ReadResult> ParseOpenApiAsync(Uri openApiFileUri, bool inlineExternal, CancellationToken cancellationToken)
+    {
+        await using var stream = await GetStreamAsync(openApiFileUri, cancellationToken: cancellationToken).ConfigureAwait(false);
+        return await ParseOpenApiAsync(stream, openApiFileUri, inlineExternal, cancellationToken).ConfigureAwait(false);
+    }
+
+    internal static async Task<ReadResult> ParseOpenApiAsync(Stream stream, Uri openApiFileUri, bool inlineExternal, CancellationToken cancellationToken)
+    {
+        ReadResult result = await new OpenApiStreamReader(new OpenApiReaderSettings
+        {
+            LoadExternalRefs = inlineExternal,
+            BaseUrl = openApiFileUri
+        }
+        ).ReadAsync(stream, cancellationToken).ConfigureAwait(false);
+
+        return result;
+    }
+
+    internal static async Task<Stream> GetStreamAsync(Uri uri, CancellationToken cancellationToken = default)
+    {
+        if (!uri.Scheme.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+            throw new ArgumentException($"The input {uri} is not a valid url", nameof(uri));
+        try
+        {
+            return await s_httpClient.Value.GetStreamAsync(uri, cancellationToken).ConfigureAwait(false);
+        }
+        catch (HttpRequestException ex)
+        {
+            throw new InvalidOperationException($"Could not download the file at {uri}", ex);
+        }
+    }
 }
 
 internal class FixedFieldMap<T> : Dictionary<string, Action<T, JsonElement>>

src/lib/Publisher.cs

@@ -13,22 +13,20 @@ public class Publisher
 
     public Publisher(string name, string contactEmail)
     {
-        Validate(name, contactEmail);
-
         Name = name;
         ContactEmail = contactEmail;
+        Validate();
     }
     private Publisher(JsonElement value)
     {
         ParsingHelpers.ParseMap(value, this, handlers);
-        Validate(Name, ContactEmail);
+        Validate();
     }
 
     // Write method
     public void Write(Utf8JsonWriter writer)
     {
-        Validate(Name, ContactEmail);
-
+        Validate();
         writer.WriteStartObject();
         writer.WriteString(NameProperty, Name);
         writer.WriteString(ContactEmailProperty, ContactEmail);
@@ -41,10 +39,10 @@ internal static Publisher Load(JsonElement value)
         return new Publisher(value);
     }
 
-    private static void Validate(string? name, string? contactEmail)
+    internal void Validate()
     {
-        ValidationHelpers.ValidateNullOrWhitespace(nameof(name), name, nameof(Publisher));
-        ValidationHelpers.ValidateEmail(nameof(contactEmail), contactEmail, nameof(Publisher));
+        ValidationHelpers.ValidateNullOrWhitespace(nameof(Name), Name, nameof(Publisher));
+        ValidationHelpers.ValidateEmail(nameof(ContactEmail), ContactEmail, nameof(Publisher));
     }
 
     private static readonly FixedFieldMap<Publisher> handlers = new()

src/lib/TypeExtensions/ApiManifestDocumentExtensions.cs

@@ -0,0 +1,78 @@
+using Microsoft.OpenApi.ApiManifest.Exceptions;
+using Microsoft.OpenApi.ApiManifest.Helpers;
+using Microsoft.OpenApi.ApiManifest.OpenAI;
+using Microsoft.OpenApi.ApiManifest.OpenAI.Authentication;
+using Microsoft.OpenApi.Models;
+
+namespace Microsoft.OpenApi.ApiManifest.TypeExtensions
+{
+    public static class ApiManifestDocumentExtensions
+    {
+        /// <summary>
+        /// Converts an instance of <see cref="ApiManifestDocument"/> to an instance of <see cref="OpenAIPluginManifest"/>.
+        /// </summary>
+        /// <param name="apiManifestDocument">A valid instance of <see cref="ApiManifestDocument"/> to generate an OpenAI Plugin manifest from.</param>
+        /// <param name="logoUrl">The URL to a logo for the plugin.</param>
+        /// <param name="legalInfoUrl">The URL to a page with legal information about the plugin.</param>
+        /// <param name="apiDependencyName">The name of apiDependency to use from the provided <see cref="ApiManifestDocument.ApiDependencies"/>. The method defaults to the first apiDependency in  <see cref="ApiManifestDocument.ApiDependencies"/> if no value is provided.</param>
+        /// <param name="openApiPath">The path to where the OpenAPI file that's packaged with the plugin manifest is stored. The method defaults to the ApiDependency.ApiDescriptionUrl if none is provided.</param>
+        /// <param name="cancellationToken">Propagates notification that operations should be canceled.</param>
+        /// <returns>A <see cref="Task{OpenAIPluginManifest}"/></returns>
+        public static async Task<OpenAIPluginManifest> ToOpenAIPluginManifestAsync(this ApiManifestDocument apiManifestDocument, string logoUrl, string legalInfoUrl, string? apiDependencyName = default, string? openApiPath = default, CancellationToken cancellationToken = default)
+        {
+            if (!TryGetApiDependency(apiManifestDocument.ApiDependencies, apiDependencyName, out ApiDependency? apiDependency))
+            {
+                throw new ApiManifestException(string.Format(ErrorMessage.ApiDependencyNotFound, nameof(OpenAIPluginManifest)));
+            }
+            else if (string.IsNullOrWhiteSpace(apiDependency?.ApiDescriptionUrl))
+            {
+                throw new ApiManifestException(string.Format(ErrorMessage.ApiDescriptionUrlNotFound, nameof(OpenAIPluginManifest)));
+            }
+            else
+            {
+                var result = await ParsingHelpers.ParseOpenApiAsync(new Uri(apiDependency.ApiDescriptionUrl), false, cancellationToken).ConfigureAwait(false);
+                return apiManifestDocument.ToOpenAIPluginManifest(result.OpenApiDocument, logoUrl, legalInfoUrl, openApiPath ?? apiDependency.ApiDescriptionUrl);
+            }
+        }
+
+        /// <summary>
+        /// Converts an instance of <see cref="ApiManifestDocument"/> to an instance of <see cref="OpenAIPluginManifest"/>.
+        /// </summary>
+        /// <param name="apiManifestDocument">A valid instance of <see cref="ApiManifestDocument"/> with at least one API dependency.</param>
+        /// <param name="openApiDocument">The OpenAPI document to use for the OpenAIPluginManifest.</param>
+        /// <param name="logoUrl">The URL to a logo for the plugin.</param>
+        /// <param name="legalInfoUrl">The URL to a page with legal information about the plugin.</param>
+        /// <param name="openApiPath">The path to where the OpenAPI file that's packaged with the plugin manifest is stored.</param>
+        /// <returns>A <see cref="OpenAIPluginManifest"/></returns>
+        public static OpenAIPluginManifest ToOpenAIPluginManifest(this ApiManifestDocument apiManifestDocument, OpenApiDocument openApiDocument, string logoUrl, string legalInfoUrl, string openApiPath)
+        {
+            // Validates the ApiManifestDocument before generating the OpenAI manifest. This includes the publisher object.
+            apiManifestDocument.Validate();
+            string contactEmail = apiManifestDocument.Publisher?.ContactEmail!;
+
+            var openApiManifest = OpenApiPluginFactory.CreateOpenAIPluginManifest(openApiDocument.Info.Title, openApiDocument.Info.Title, logoUrl, contactEmail, legalInfoUrl, openApiDocument.Info.Version);
+            openApiManifest.Api = new Api("openapi", openApiPath);
+            openApiManifest.Auth = new ManifestNoAuth();
+            openApiManifest.DescriptionForHuman = openApiDocument.Info.Description ?? $"Description for {openApiManifest.NameForHuman}.";
+            openApiManifest.DescriptionForModel = openApiManifest.DescriptionForHuman;
+
+            return openApiManifest;
+        }
+
+        /// <summary>
+        /// Tries to get an <see cref="ApiDependency"/> from the provided <see cref="ApiDependencies"/>.
+        /// </summary>
+        /// <param name="apiDependencies">The <see cref="ApiDependencies"/> to search for the apiDependency.</param>
+        /// <param name="apiDependencyName">The name of apiDependency to use from the provided <see cref="ApiManifestDocument.ApiDependencies"/>. The method defaults to the first apiDependency in <see cref="ApiManifestDocument.ApiDependencies"/> if no value is provided.</param>
+        /// <param name="apiDependency">The <see cref="ApiDependency"/> that was found.</param>
+        /// <returns>Returns true if the apiDependency is found and not null, otherwise false.</returns>
+        private static bool TryGetApiDependency(ApiDependencies apiDependencies, string? apiDependencyName, out ApiDependency? apiDependency)
+        {
+            if (string.IsNullOrEmpty(apiDependencyName))
+                apiDependency = apiDependencies.FirstOrDefault().Value;
+            else
+                _ = apiDependencies.TryGetValue(apiDependencyName, out apiDependency);
+            return apiDependency != null;
+        }
+    }
+}

src/lib/apimanifest.csproj

@@ -22,4 +22,8 @@
     <AssemblyOriginatorKeyFile>..\sgKey.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.OpenApi.Readers" Version="1.6.9" />
+  </ItemGroup>
+
 </Project>

tests/ApiManifest.Tests/ApiManifest.Tests.csproj

@@ -33,4 +33,13 @@
     <ProjectReference Include="..\..\src\lib\apimanifest.csproj" />
   </ItemGroup>
 
-</Project>
+  <ItemGroup>
+    <None Update="TestFiles\exampleApiManifest.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="TestFiles\testOpenApi.yaml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

tests/ApiManifest.Tests/Helpers/ParsingHelpersTests.cs

@@ -112,5 +112,29 @@ public void ParseKeyValuePair()
             var kvPairs = ParsingHelpers.ParseKey(exampleKeyValuePair);
             Assert.Equal(3, kvPairs.Count());
         }
+
+        [Fact]
+        public async Task ParseOpenApiAsync()
+        {
+            var testOpenApiFilePath = Path.Combine(".", "TestFiles", "testOpenApi.yaml");
+            using var stream = File.OpenRead(testOpenApiFilePath);
+            var results = await ParsingHelpers.ParseOpenApiAsync(stream, new Uri("https://contoso.com/openapi.yaml"), false, CancellationToken.None);
+            Assert.Empty(results.OpenApiDiagnostic.Errors);
+            Assert.NotNull(results.OpenApiDocument);
+        }
+
+        [Fact]
+        public void ParseOpenApiWithWrongOpenApiUrl()
+        {
+            var openApiUri = new Uri("https://contoso.com/NotValid.yaml");
+            _ = Assert.ThrowsAsync<InvalidOperationException>(async () => await ParsingHelpers.ParseOpenApiAsync(openApiUri, false, CancellationToken.None));
+        }
+
+        [Fact]
+        public void ParseOpenApiWithOpenApiUrlWithAnInvalidSchema()
+        {
+            var openApiUri = new Uri("xyx://contoso.com/openapi.yaml");
+            _ = Assert.ThrowsAsync<ArgumentException>(async () => await ParsingHelpers.ParseOpenApiAsync(openApiUri, false, CancellationToken.None));
+        }
     }
 }