Merge branch 'vnext' into mk/fix-parsing-errors

MaggieKimani1 · web-flow · commit 4890e0e0ccc1 · 2022-02-15T18:55:46.000+03:00
diff --git a/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj b/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj
@@ -5,6 +5,7 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <LangVersion>9.0</LangVersion>
     <PackAsTool>true</PackAsTool>
+    <PackageIconUrl>http://go.microsoft.com/fwlink/?LinkID=288890</PackageIconUrl>
     <PackageProjectUrl>https://github.com/Microsoft/OpenAPI.NET</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
@@ -15,6 +16,7 @@
     <ToolCommandName>hidi</ToolCommandName>
     <PackageOutputPath>./../../artifacts</PackageOutputPath>
     <Version>0.5.0-preview4</Version>
+    <Description>OpenAPI.NET CLI tool for slicing OpenAPI documents</Description>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
     <PackageTags>OpenAPI .NET</PackageTags>
     <RepositoryUrl>https://github.com/Microsoft/OpenAPI.NET</RepositoryUrl>
@@ -34,6 +36,8 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="6.0.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta2.21617.1" />
+    <PackageReference Include="Microsoft.OData.Edm" Version="7.9.4" />
+    <PackageReference Include="Microsoft.OpenApi.OData" Version="1.0.9" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs
@@ -13,8 +13,11 @@
 using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using System.Xml.Linq;
+using Microsoft.OData.Edm.Csdl;
 using Microsoft.OpenApi.Extensions;
 using Microsoft.OpenApi.Models;
+using Microsoft.OpenApi.OData;
 using Microsoft.OpenApi.Readers;
 using Microsoft.OpenApi.Services;
 using Microsoft.OpenApi.Validations;
@@ -26,6 +29,7 @@ public class OpenApiService
     {
         public static async void ProcessOpenApiDocument(
             string openapi,
+            string csdl,
             FileInfo output,
             OpenApiSpecVersion? version,
             OpenApiFormat? format,
@@ -41,9 +45,9 @@ string filterbycollection
 
             try
             {
-                if (string.IsNullOrEmpty(openapi))
+                if (string.IsNullOrEmpty(openapi) && string.IsNullOrEmpty(csdl))
                 {
-                    throw new ArgumentNullException(nameof(openapi));
+                    throw new ArgumentNullException("Please input a file path");
                 }
             }
             catch (ArgumentNullException ex)
@@ -75,36 +79,56 @@ string filterbycollection
                 logger.LogError(ex.Message);
                 return;
             }
-            
-            var stream = await GetStream(openapi, logger);
 
-            // Parsing OpenAPI file
+            Stream stream;
+            OpenApiDocument document;
+            OpenApiFormat openApiFormat;
             var stopwatch = new Stopwatch();
-            stopwatch.Start();
-            logger.LogTrace("Parsing OpenApi file");
-            var result = new OpenApiStreamReader(new OpenApiReaderSettings
-            {
-                ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences,
-                RuleSet = ValidationRuleSet.GetDefaultRuleSet()
-            }
-            ).ReadAsync(stream).GetAwaiter().GetResult();
-            var document = result.OpenApiDocument;
-            stopwatch.Stop();
 
-            var context = result.OpenApiDiagnostic;
-            if (context.Errors.Count > 0)
+            if (!string.IsNullOrEmpty(csdl))
             {
-                var errorReport = new StringBuilder();
+                // Default to yaml and OpenApiVersion 3 during csdl to OpenApi conversion
+                openApiFormat = format ?? GetOpenApiFormat(csdl, logger);
+                version ??= OpenApiSpecVersion.OpenApi3_0;
 
-                foreach (var error in context.Errors)
-                {
-                    errorReport.AppendLine(error.ToString());
-                }
-                logger.LogError($"{stopwatch.ElapsedMilliseconds}ms: OpenApi Parsing errors {string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())}");
+                stream = await GetStream(csdl, logger);
+                document = ConvertCsdlToOpenApi(stream);
             }
             else
             {
-                logger.LogTrace("{timestamp}ms: Parsed OpenApi successfully. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count);
+                stream = await GetStream(openapi, logger);
+
+                // Parsing OpenAPI file
+                stopwatch.Start();
+                logger.LogTrace("Parsing OpenApi file");
+                var result = new OpenApiStreamReader(new OpenApiReaderSettings
+                {
+                    ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences,
+                    RuleSet = ValidationRuleSet.GetDefaultRuleSet()
+                }
+                ).ReadAsync(stream).GetAwaiter().GetResult();
+
+                document = result.OpenApiDocument;
+                stopwatch.Stop();
+
+                var context = result.OpenApiDiagnostic;
+                if (context.Errors.Count > 0)
+                {
+                    var errorReport = new StringBuilder();
+
+                    foreach (var error in context.Errors)
+                    {
+                        errorReport.AppendLine(error.ToString());
+                    }
+                    logger.LogError($"{stopwatch.ElapsedMilliseconds}ms: OpenApi Parsing errors {string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())}");
+                }
+                else
+                {
+                    logger.LogTrace("{timestamp}ms: Parsed OpenApi successfully. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count);
+                }
+
+                openApiFormat = format ?? GetOpenApiFormat(openapi, logger);
+                version ??= result.OpenApiDiagnostic.SpecificationVersion;
             }
 
             Func<string, OperationType?, OpenApiOperation, bool> predicate;
@@ -151,8 +175,6 @@ string filterbycollection
                 ReferenceInline = inline ? ReferenceInlineSetting.InlineLocalReferences : ReferenceInlineSetting.DoNotInlineReferences
             };
 
-            var openApiFormat = format ?? GetOpenApiFormat(openapi, logger);
-            var openApiVersion = version ?? result.OpenApiDiagnostic.SpecificationVersion;
             IOpenApiWriter writer = openApiFormat switch
             {
                 OpenApiFormat.Json => new OpenApiJsonWriter(textWriter, settings),
@@ -163,14 +185,62 @@ string filterbycollection
             logger.LogTrace("Serializing to OpenApi document using the provided spec version and writer");
             
             stopwatch.Start();
-            document.Serialize(writer, openApiVersion);
+            document.Serialize(writer, (OpenApiSpecVersion)version);
             stopwatch.Stop();
 
             logger.LogTrace($"Finished serializing in {stopwatch.ElapsedMilliseconds}ms");
 
             textWriter.Flush();
         }
 
+        /// <summary>
+        /// Converts CSDL to OpenAPI
+        /// </summary>
+        /// <param name="csdl">The CSDL stream.</param>
+        /// <returns>An OpenAPI document.</returns>
+        public static OpenApiDocument ConvertCsdlToOpenApi(Stream csdl)
+        {
+            using var reader = new StreamReader(csdl);
+            var csdlText = reader.ReadToEndAsync().GetAwaiter().GetResult();           
+            var edmModel = CsdlReader.Parse(XElement.Parse(csdlText).CreateReader());
+
+            var settings = new OpenApiConvertSettings()
+            {
+                EnableKeyAsSegment = true,
+                EnableOperationId = true,
+                PrefixEntityTypeNameBeforeKey = true,
+                TagDepth = 2,
+                EnablePagination = true,
+                EnableDiscriminatorValue = false,
+                EnableDerivedTypesReferencesForRequestBody = false,
+                EnableDerivedTypesReferencesForResponses = false,
+                ShowRootPath = true,
+                ShowLinks = true
+            };
+            OpenApiDocument document = edmModel.ConvertToOpenApi(settings);
+
+            document = FixReferences(document);
+
+            return document;
+        }
+
+        /// <summary>
+        /// Fixes the references in the resulting OpenApiDocument.
+        /// </summary>
+        /// <param name="document"> The converted OpenApiDocument.</param>
+        /// <returns> A valid OpenApiDocument instance.</returns>
+        public static OpenApiDocument FixReferences(OpenApiDocument document)
+        {
+            // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance.
+            // So we write it out, and read it back in again to fix it up.
+
+            var sb = new StringBuilder();
+            document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb)));
+            var doc = new OpenApiStringReader().Read(sb.ToString(), out _);
+
+            return doc;
+        }
+
         private static async Task<Stream> GetStream(string input, ILogger logger)
         {
             var stopwatch = new Stopwatch();
@@ -286,10 +356,10 @@ internal static async void ValidateOpenApiDocument(string openapi, LogLevel logl
             Console.WriteLine(statsVisitor.GetStatisticsReport());
         }
 
-        private static OpenApiFormat GetOpenApiFormat(string openapi, ILogger logger)
+        private static OpenApiFormat GetOpenApiFormat(string input, ILogger logger)
         {
             logger.LogTrace("Getting the OpenApi format");
-            return !openapi.StartsWith("http") && Path.GetExtension(openapi) == ".json" ? OpenApiFormat.Json : OpenApiFormat.Yaml;
+            return !input.StartsWith("http") && Path.GetExtension(input) == ".json" ? OpenApiFormat.Json : OpenApiFormat.Yaml;
         }
 
         private static ILogger ConfigureLoggerInstance(LogLevel loglevel)
diff --git a/src/Microsoft.OpenApi.Hidi/Program.cs b/src/Microsoft.OpenApi.Hidi/Program.cs
@@ -19,6 +19,9 @@ static async Task<int> Main(string[] args)
             var descriptionOption = new Option<string>("--openapi", "Input OpenAPI description file path or URL");
             descriptionOption.AddAlias("-d");
 
+            var csdlOption = new Option<string>("--csdl", "Input CSDL file path or URL");
+            csdlOption.AddAlias("-cs");
+
             var outputOption = new Option<FileInfo>("--output", () => new FileInfo("./output"), "The output directory path for the generated file.") { Arity = ArgumentArity.ZeroOrOne };
             outputOption.AddAlias("-o");
 
@@ -57,6 +60,7 @@ static async Task<int> Main(string[] args)
             var transformCommand = new Command("transform")
             {
                 descriptionOption,
+                csdlOption,
                 outputOption,
                 versionOption,
                 formatOption,
@@ -68,8 +72,8 @@ static async Task<int> Main(string[] args)
                 resolveExternalOption,
             };
 
-            transformCommand.SetHandler<string, FileInfo, OpenApiSpecVersion?, OpenApiFormat?, LogLevel, bool, bool, string, string, string> (
-                OpenApiService.ProcessOpenApiDocument, descriptionOption, outputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption);
+            transformCommand.SetHandler<string, string, FileInfo, OpenApiSpecVersion?, OpenApiFormat?, LogLevel, bool, bool, string, string, string> (
+                OpenApiService.ProcessOpenApiDocument, descriptionOption, csdlOption, outputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption);
 
             rootCommand.Add(transformCommand);
             rootCommand.Add(validateCommand);
diff --git a/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj b/src/Microsoft.OpenApi/Microsoft.OpenApi.csproj
@@ -38,7 +38,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
-        <PackageReference Include="System.Text.Json" Version="6.0.1" />
+        <PackageReference Include="System.Text.Json" Version="6.0.2" />
     </ItemGroup>
 
     <ItemGroup>
diff --git a/src/Microsoft.OpenApi/Validations/Rules/OpenApiParameterRules.cs b/src/Microsoft.OpenApi/Validations/Rules/OpenApiParameterRules.cs
@@ -103,7 +103,8 @@ public static class OpenApiParameterRules
             new ValidationRule<OpenApiParameter>(
                 (context, parameter) =>
                 {
-                    if (parameter.In == ParameterLocation.Path && !context.PathString.Contains("{" + parameter.Name + "}"))
+                    if (parameter.In == ParameterLocation.Path && 
+                           !(context.PathString.Contains("{" + parameter.Name + "}") || context.PathString.Contains("#/components")))
                     {
                         context.Enter("in");
                         context.CreateError(
diff --git a/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj b/test/Microsoft.OpenApi.Tests/Microsoft.OpenApi.Tests.csproj
@@ -20,8 +20,8 @@
     <PackageReference Include="Moq" Version="4.16.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="SharpYaml" Version="1.8.0" />
-    <PackageReference Include="Verify" Version="15.2.1" />
-    <PackageReference Include="Verify.Xunit" Version="14.14.1" />
+    <PackageReference Include="Verify" Version="16.1.1" />
+    <PackageReference Include="Verify.Xunit" Version="16.1.1" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>
@@ -44,5 +44,8 @@
     <None Update="UtilityFiles\postmanCollection_ver2.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="UtilityFiles\Todo.xml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 </Project>
diff --git a/test/Microsoft.OpenApi.Tests/Services/OpenApiServiceTests.cs b/test/Microsoft.OpenApi.Tests/Services/OpenApiServiceTests.cs
@@ -0,0 +1,54 @@
+﻿// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.IO;
+using Microsoft.OpenApi.Hidi;
+using Microsoft.OpenApi.Services;
+using Xunit;
+
+namespace Microsoft.OpenApi.Tests.Services
+{
+    public class OpenApiServiceTests
+    {
+        [Fact]
+        public void ReturnConvertedCSDLFile()
+        {
+            // Arrange
+            var filePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "UtilityFiles\\Todo.xml");
+            var fileInput = new FileInfo(filePath);
+            var csdlStream = fileInput.OpenRead();
+
+            // Act
+            var openApiDoc = OpenApiService.ConvertCsdlToOpenApi(csdlStream);
+            var expectedPathCount = 5;
+
+            // Assert
+            Assert.NotNull(openApiDoc);
+            Assert.NotEmpty(openApiDoc.Paths);
+            Assert.Equal(openApiDoc.Paths.Count, expectedPathCount);
+        }
+        
+        [Theory]
+        [InlineData("Todos.Todo.UpdateTodo",null, 1)]
+        [InlineData("Todos.Todo.ListTodo",null, 1)]
+        [InlineData(null, "Todos.Todo", 4)]
+        public void ReturnFilteredOpenApiDocBasedOnOperationIdsAndInputCsdlDocument(string operationIds, string tags, int expectedPathCount)
+        {
+            // Arrange
+            var filePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "UtilityFiles\\Todo.xml");
+            var fileInput = new FileInfo(filePath);
+            var csdlStream = fileInput.OpenRead();
+
+            // Act
+            var openApiDoc = OpenApiService.ConvertCsdlToOpenApi(csdlStream);
+            var predicate = OpenApiFilterService.CreatePredicate(operationIds, tags);
+            var subsetOpenApiDocument = OpenApiFilterService.CreateFilteredDocument(openApiDoc, predicate);
+
+            // Assert
+            Assert.NotNull(subsetOpenApiDocument);
+            Assert.NotEmpty(subsetOpenApiDocument.Paths);
+            Assert.Equal(expectedPathCount, subsetOpenApiDocument.Paths.Count);
+        }
+    }
+}
diff --git a/test/Microsoft.OpenApi.Tests/UtilityFiles/Todo.xml b/test/Microsoft.OpenApi.Tests/UtilityFiles/Todo.xml
@@ -0,0 +1,21 @@
+﻿<edmx:Edmx xmlns:edmx="http://docs.oasis-open.org/odata/ns/edmx" Version="4.0">
+    <edmx:DataServices>
+        <Schema xmlns="http://docs.oasis-open.org/odata/ns/edm" Namespace="microsoft.graph" >
+
+            <EntityContainer Name="TodoService">
+                <EntitySet Name="Todos" EntityType="microsoft.graph.Todo">
+                </EntitySet>
+            </EntityContainer>
+
+            <EntityType Name="Todo" HasStream="true">
+                <Key>
+                    <PropertyRef Name="Id"/>
+                </Key>
+                <Property Name="Id" Type="Edm.String"/>
+                <Property Name="Logo" Type="Edm.Stream"/>
+                <Property Name="Description" Type="Edm.String"/>
+            </EntityType>
+
+        </Schema>
+    </edmx:DataServices>
+</edmx:Edmx>

Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,8 @@ public static class OpenApiParameterRules`
`103`	`103`	`new ValidationRule<OpenApiParameter>(`
`104`	`104`	`(context, parameter) =>`
`105`	`105`	`{`
`106`		`- if (parameter.In == ParameterLocation.Path && !context.PathString.Contains("{" + parameter.Name + "}"))`
	`106`	`+ if (parameter.In == ParameterLocation.Path &&`
	`107`	`+ !(context.PathString.Contains("{" + parameter.Name + "}") \|\| context.PathString.Contains("#/components")))`
`107`	`108`	`{`
`108`	`109`	`context.Enter("in");`
`109`	`110`	`context.CreateError(`