diff --git a/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj b/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj index 4db249c8f..9fe37bbc2 100644 --- a/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj +++ b/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj @@ -3,14 +3,19 @@ Exe netcoreapp3.1 + 9.0 true hidi ./../../artifacts - 0.5.0-preview2 + 0.5.0-preview3 - + + + + + diff --git a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs index abef3617f..3c9fdb7d5 100644 --- a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs +++ b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs @@ -3,12 +3,16 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Security; using System.Text; using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.OpenApi.Extensions; using Microsoft.OpenApi.Models; using Microsoft.OpenApi.Readers; @@ -18,91 +22,136 @@ namespace Microsoft.OpenApi.Hidi { - public static class OpenApiService + public class OpenApiService { - public static void ProcessOpenApiDocument( - string input, + public static async void ProcessOpenApiDocument( + string openapi, FileInfo output, OpenApiSpecVersion? version, OpenApiFormat? format, - string filterByOperationIds, - string filterByTags, - string filterByCollection, + LogLevel loglevel, bool inline, - bool resolveExternal) + bool resolveexternal, + string filterbyoperationids, + string filterbytags, + string filterbycollection + ) { - if (string.IsNullOrEmpty(input)) + var logger = ConfigureLoggerInstance(loglevel); + + try { - throw new ArgumentNullException(nameof(input)); + if (string.IsNullOrEmpty(openapi)) + { + throw new ArgumentNullException(nameof(openapi)); + } } - if(output == null) + catch (ArgumentNullException ex) { - throw new ArgumentException(nameof(output)); + logger.LogError(ex.Message); + return; } - if (output.Exists) + try { - throw new IOException("The file you're writing to already exists. Please input a new output path."); + if(output == null) + { + throw new ArgumentException(nameof(output)); + } } + catch (ArgumentException ex) + { + logger.LogError(ex.Message); + return; + } + try + { + if (output.Exists) + { + throw new IOException("The file you're writing to already exists. Please input a new file path."); + } + } + catch (IOException ex) + { + logger.LogError(ex.Message); + return; + } + + var stream = await GetStream(openapi, logger); - var stream = GetStream(input); + // Parsing OpenAPI file + var stopwatch = new Stopwatch(); + stopwatch.Start(); + logger.LogTrace("Parsing OpenApi file"); var result = new OpenApiStreamReader(new OpenApiReaderSettings { - ReferenceResolution = resolveExternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences, + ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences, RuleSet = ValidationRuleSet.GetDefaultRuleSet() } ).ReadAsync(stream).GetAwaiter().GetResult(); - var document = result.OpenApiDocument; + stopwatch.Stop(); + + var context = result.OpenApiDiagnostic; + if (context.Errors.Count > 0) + { + var errorReport = new StringBuilder(); + + foreach (var error in context.Errors) + { + errorReport.AppendLine(error.ToString()); + } + logger.LogError($"{stopwatch.ElapsedMilliseconds}ms: OpenApi Parsing errors {string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())}"); + } + else + { + logger.LogTrace("{timestamp}ms: Parsed OpenApi successfully. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count); + } + Func predicate; - // Check if filter options are provided, then execute - if (!string.IsNullOrEmpty(filterByOperationIds) && !string.IsNullOrEmpty(filterByTags)) + // Check if filter options are provided, then slice the OpenAPI document + if (!string.IsNullOrEmpty(filterbyoperationids) && !string.IsNullOrEmpty(filterbytags)) { throw new InvalidOperationException("Cannot filter by operationIds and tags at the same time."); } - if (!string.IsNullOrEmpty(filterByOperationIds)) + if (!string.IsNullOrEmpty(filterbyoperationids)) { - predicate = OpenApiFilterService.CreatePredicate(operationIds: filterByOperationIds); + logger.LogTrace("Creating predicate based on the operationIds supplied."); + predicate = OpenApiFilterService.CreatePredicate(operationIds: filterbyoperationids); + + logger.LogTrace("Creating subset OpenApi document."); document = OpenApiFilterService.CreateFilteredDocument(document, predicate); } - if (!string.IsNullOrEmpty(filterByTags)) + if (!string.IsNullOrEmpty(filterbytags)) { - predicate = OpenApiFilterService.CreatePredicate(tags: filterByTags); - document = OpenApiFilterService.CreateFilteredDocument(document, predicate); - } + logger.LogTrace("Creating predicate based on the tags supplied."); + predicate = OpenApiFilterService.CreatePredicate(tags: filterbytags); - if (!string.IsNullOrEmpty(filterByCollection)) - { - var fileStream = GetStream(filterByCollection); - var requestUrls = ParseJsonCollectionFile(fileStream); - predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source:document); + logger.LogTrace("Creating subset OpenApi document."); document = OpenApiFilterService.CreateFilteredDocument(document, predicate); } - - var context = result.OpenApiDiagnostic; - - if (context.Errors.Count > 0) + if (!string.IsNullOrEmpty(filterbycollection)) { - var errorReport = new StringBuilder(); + var fileStream = await GetStream(filterbycollection, logger); + var requestUrls = ParseJsonCollectionFile(fileStream, logger); - foreach (var error in context.Errors) - { - errorReport.AppendLine(error.ToString()); - } + logger.LogTrace("Creating predicate based on the paths and Http methods defined in the Postman collection."); + predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source:document); - throw new ArgumentException(string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())); + logger.LogTrace("Creating subset OpenApi document."); + document = OpenApiFilterService.CreateFilteredDocument(document, predicate); } - + + logger.LogTrace("Creating a new file"); using var outputStream = output?.Create(); - - var textWriter = outputStream != null ? new StreamWriter(outputStream) : Console.Out; + var textWriter = outputStream != null ? new StreamWriter(outputStream) : Console.Out; var settings = new OpenApiWriterSettings() { ReferenceInline = inline ? ReferenceInlineSetting.InlineLocalReferences : ReferenceInlineSetting.DoNotInlineReferences }; - var openApiFormat = format ?? GetOpenApiFormat(input); + var openApiFormat = format ?? GetOpenApiFormat(openapi, logger); var openApiVersion = version ?? result.OpenApiDiagnostic.SpecificationVersion; IOpenApiWriter writer = openApiFormat switch { @@ -110,31 +159,65 @@ public static void ProcessOpenApiDocument( OpenApiFormat.Yaml => new OpenApiYamlWriter(textWriter, settings), _ => throw new ArgumentException("Unknown format"), }; + + logger.LogTrace("Serializing to OpenApi document using the provided spec version and writer"); + + stopwatch.Start(); document.Serialize(writer, openApiVersion); + stopwatch.Stop(); + + logger.LogTrace($"Finished serializing in {stopwatch.ElapsedMilliseconds}ms"); textWriter.Flush(); } - private static Stream GetStream(string input) + private static async Task GetStream(string input, ILogger logger) { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + Stream stream; if (input.StartsWith("http")) { - var httpClient = new HttpClient(new HttpClientHandler() + try { - SslProtocols = System.Security.Authentication.SslProtocols.Tls12, - }) + using var httpClientHandler = new HttpClientHandler() + { + SslProtocols = System.Security.Authentication.SslProtocols.Tls12, + }; + using var httpClient = new HttpClient(httpClientHandler) + { + DefaultRequestVersion = HttpVersion.Version20 + }; + stream = await httpClient.GetStreamAsync(input); + } + catch (HttpRequestException ex) { - DefaultRequestVersion = HttpVersion.Version20 - }; - stream = httpClient.GetStreamAsync(input).Result; + logger.LogError($"Could not download the file at {input}, reason{ex}"); + return null; + } } else { - var fileInput = new FileInfo(input); - stream = fileInput.OpenRead(); + try + { + var fileInput = new FileInfo(input); + stream = fileInput.OpenRead(); + } + catch (Exception ex) when (ex is FileNotFoundException || + ex is PathTooLongException || + ex is DirectoryNotFoundException || + ex is IOException || + ex is UnauthorizedAccessException || + ex is SecurityException || + ex is NotSupportedException) + { + logger.LogError($"Could not open the file at {input}, reason: {ex.Message}"); + return null; + } } - + stopwatch.Stop(); + logger.LogTrace("{timestamp}ms: Read file {input}", stopwatch.ElapsedMilliseconds, input); return stream; } @@ -143,11 +226,11 @@ private static Stream GetStream(string input) /// /// A file stream. /// A dictionary of request urls and http methods from a collection. - public static Dictionary> ParseJsonCollectionFile(Stream stream) + public static Dictionary> ParseJsonCollectionFile(Stream stream, ILogger logger) { var requestUrls = new Dictionary>(); - // Convert file to JsonDocument + logger.LogTrace("Parsing the json collection file into a JsonDocument"); using var document = JsonDocument.Parse(stream); var root = document.RootElement; var itemElement = root.GetProperty("item"); @@ -166,21 +249,21 @@ public static Dictionary> ParseJsonCollectionFile(Stream st requestUrls[path].Add(method); } } - + logger.LogTrace("Finished fetching the list of paths and Http methods defined in the Postman collection."); return requestUrls; } - internal static void ValidateOpenApiDocument(string input) + internal static async void ValidateOpenApiDocument(string openapi, LogLevel loglevel) { - if (input == null) + if (string.IsNullOrEmpty(openapi)) { - throw new ArgumentNullException("input"); + throw new ArgumentNullException(nameof(openapi)); } - - var stream = GetStream(input); + var logger = ConfigureLoggerInstance(loglevel); + var stream = await GetStream(openapi, logger); OpenApiDocument document; - + logger.LogTrace("Parsing the OpenApi file"); document = new OpenApiStreamReader(new OpenApiReaderSettings { RuleSet = ValidationRuleSet.GetDefaultRuleSet() @@ -199,12 +282,33 @@ internal static void ValidateOpenApiDocument(string input) var walker = new OpenApiWalker(statsVisitor); walker.Walk(document); + logger.LogTrace("Finished walking through the OpenApi document. Generating a statistics report.."); Console.WriteLine(statsVisitor.GetStatisticsReport()); } - private static OpenApiFormat GetOpenApiFormat(string input) + private static OpenApiFormat GetOpenApiFormat(string openapi, ILogger logger) + { + logger.LogTrace("Getting the OpenApi format"); + return !openapi.StartsWith("http") && Path.GetExtension(openapi) == ".json" ? OpenApiFormat.Json : OpenApiFormat.Yaml; + } + + private static ILogger ConfigureLoggerInstance(LogLevel loglevel) { - return !input.StartsWith("http") && Path.GetExtension(input) == ".json" ? OpenApiFormat.Json : OpenApiFormat.Yaml; + // Configure logger options + #if DEBUG + loglevel = loglevel > LogLevel.Debug ? LogLevel.Debug : loglevel; + #endif + + var logger = LoggerFactory.Create((builder) => { + builder + .AddConsole() + #if DEBUG + .AddDebug() + #endif + .SetMinimumLevel(loglevel); + }).CreateLogger(); + + return logger; } } } diff --git a/src/Microsoft.OpenApi.Hidi/Program.cs b/src/Microsoft.OpenApi.Hidi/Program.cs index b3752ef97..841c710e5 100644 --- a/src/Microsoft.OpenApi.Hidi/Program.cs +++ b/src/Microsoft.OpenApi.Hidi/Program.cs @@ -2,9 +2,9 @@ // Licensed under the MIT license. using System.CommandLine; -using System.CommandLine.Invocation; using System.IO; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Microsoft.OpenApi.Hidi { @@ -15,26 +15,61 @@ static async Task Main(string[] args) var rootCommand = new RootCommand() { }; + // command option parameters and aliases + var descriptionOption = new Option("--openapi", "Input OpenAPI description file path or URL"); + descriptionOption.AddAlias("-d"); + + var outputOption = new Option("--output", () => new FileInfo("./output"), "The output directory path for the generated file.") { Arity = ArgumentArity.ZeroOrOne }; + outputOption.AddAlias("-o"); + + var versionOption = new Option("--version", "OpenAPI specification version"); + versionOption.AddAlias("-v"); + + var formatOption = new Option("--format", "File format"); + formatOption.AddAlias("-f"); + + var logLevelOption = new Option("--loglevel", () => LogLevel.Warning, "The log level to use when logging messages to the main output."); + logLevelOption.AddAlias("-ll"); + + var filterByOperationIdsOption = new Option("--filter-by-operationids", "Filters OpenApiDocument by OperationId(s) provided"); + filterByOperationIdsOption.AddAlias("-op"); + + var filterByTagsOption = new Option("--filter-by-tags", "Filters OpenApiDocument by Tag(s) provided"); + filterByTagsOption.AddAlias("-t"); + + var filterByCollectionOption = new Option("--filter-by-collection", "Filters OpenApiDocument by Postman collection provided"); + filterByCollectionOption.AddAlias("-c"); + + var inlineOption = new Option("--inline", "Inline $ref instances"); + inlineOption.AddAlias("-i"); + + var resolveExternalOption = new Option("--resolve-external", "Resolve external $refs"); + resolveExternalOption.AddAlias("-ex"); + var validateCommand = new Command("validate") { - new Option("--input", "Input OpenAPI description file path or URL", typeof(string) ) + descriptionOption, + logLevelOption }; - validateCommand.Handler = CommandHandler.Create(OpenApiService.ValidateOpenApiDocument); + + validateCommand.SetHandler(OpenApiService.ValidateOpenApiDocument, descriptionOption, logLevelOption); var transformCommand = new Command("transform") { - new Option("--input", "Input OpenAPI description file path or URL", typeof(string) ), - new Option("--output","Output OpenAPI description file", typeof(FileInfo), arity: ArgumentArity.ZeroOrOne), - new Option("--version", "OpenAPI specification version", typeof(OpenApiSpecVersion)), - new Option("--format", "File format",typeof(OpenApiFormat) ), - new Option("--inline", "Inline $ref instances", typeof(bool) ), - new Option("--resolveExternal","Resolve external $refs", typeof(bool)), - new Option("--filterByOperationIds", "Filters OpenApiDocument by OperationId(s) provided", typeof(string)), - new Option("--filterByTags", "Filters OpenApiDocument by Tag(s) provided", typeof(string)), - new Option("--filterByCollection", "Filters OpenApiDocument by Postman collection provided", typeof(string)) + descriptionOption, + outputOption, + versionOption, + formatOption, + logLevelOption, + filterByOperationIdsOption, + filterByTagsOption, + filterByCollectionOption, + inlineOption, + resolveExternalOption, }; - transformCommand.Handler = CommandHandler.Create( - OpenApiService.ProcessOpenApiDocument); + + transformCommand.SetHandler ( + OpenApiService.ProcessOpenApiDocument, descriptionOption, outputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption); rootCommand.Add(transformCommand); rootCommand.Add(validateCommand); diff --git a/src/Microsoft.OpenApi.Hidi/appsettings.json b/src/Microsoft.OpenApi.Hidi/appsettings.json new file mode 100644 index 000000000..882248cf8 --- /dev/null +++ b/src/Microsoft.OpenApi.Hidi/appsettings.json @@ -0,0 +1,7 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug" + } + } +} \ No newline at end of file diff --git a/src/Microsoft.OpenApi.Readers/Microsoft.OpenApi.Readers.csproj b/src/Microsoft.OpenApi.Readers/Microsoft.OpenApi.Readers.csproj index a594df10d..2f6bc75b9 100644 --- a/src/Microsoft.OpenApi.Readers/Microsoft.OpenApi.Readers.csproj +++ b/src/Microsoft.OpenApi.Readers/Microsoft.OpenApi.Readers.csproj @@ -10,7 +10,7 @@ Microsoft Microsoft.OpenApi.Readers Microsoft.OpenApi.Readers - 1.3.1-preview2 + 1.3.1-preview3 OpenAPI.NET Readers for JSON and YAML documents © Microsoft Corporation. All rights reserved. OpenAPI .NET diff --git a/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj b/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj index d2839edc7..388cf45e2 100644 --- a/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj +++ b/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj @@ -11,7 +11,7 @@ Microsoft Microsoft.OpenApi Microsoft.OpenApi - 1.3.1-preview2 + 1.3.1-preview3 .NET models with JSON and YAML writers for OpenAPI specification © Microsoft Corporation. All rights reserved. OpenAPI .NET diff --git a/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj b/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj index 13feb0bc9..7ed607fd3 100644 --- a/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj +++ b/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj @@ -243,7 +243,7 @@ - + diff --git a/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj b/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj index 56bdd0983..d8fd47fd1 100644 --- a/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj +++ b/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj @@ -15,11 +15,12 @@ - + + - + diff --git a/test/Microsoft.OpenApi.Tests/Services/OpenApiFilterServiceTests.cs b/test/Microsoft.OpenApi.Tests/Services/OpenApiFilterServiceTests.cs index f470b8577..78f8ec048 100644 --- a/test/Microsoft.OpenApi.Tests/Services/OpenApiFilterServiceTests.cs +++ b/test/Microsoft.OpenApi.Tests/Services/OpenApiFilterServiceTests.cs @@ -3,10 +3,12 @@ using System; using System.IO; +using Microsoft.Extensions.Logging; using Microsoft.OpenApi.Hidi; using Microsoft.OpenApi.Models; using Microsoft.OpenApi.Services; using Microsoft.OpenApi.Tests.UtilityFiles; +using Moq; using Xunit; namespace Microsoft.OpenApi.Tests.Services @@ -14,10 +16,14 @@ namespace Microsoft.OpenApi.Tests.Services public class OpenApiFilterServiceTests { private readonly OpenApiDocument _openApiDocumentMock; + private readonly Mock> _mockLogger; + private readonly ILogger _logger; public OpenApiFilterServiceTests() { _openApiDocumentMock = OpenApiDocumentMock.CreateOpenApiDocument(); + _mockLogger = new Mock>(); + _logger = _mockLogger.Object; } [Theory] @@ -53,7 +59,7 @@ public void ReturnFilteredOpenApiDocumentBasedOnPostmanCollection() var stream = fileInput.OpenRead(); // Act - var requestUrls = OpenApiService.ParseJsonCollectionFile(stream); + var requestUrls = OpenApiService.ParseJsonCollectionFile(stream, _logger); var predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source: _openApiDocumentMock); var subsetOpenApiDocument = OpenApiFilterService.CreateFilteredDocument(_openApiDocumentMock, predicate); @@ -72,7 +78,7 @@ public void ThrowsExceptionWhenUrlsInCollectionAreMissingFromSourceDocument() var stream = fileInput.OpenRead(); // Act - var requestUrls = OpenApiService.ParseJsonCollectionFile(stream); + var requestUrls = OpenApiService.ParseJsonCollectionFile(stream, _logger); // Assert var message = Assert.Throws(() =>