Resolved merge conflicts with vnext

Author: darrelmiller

src/Microsoft.OpenApi.Hidi/OpenApiService.cs

@@ -10,8 +10,8 @@
 using System.Net.Http;
 using System.Security;
 using System.Text;
-using System.Text.Json;
 using System.Threading.Tasks;
+using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using System.Xml.Linq;
 using Microsoft.OData.Edm.Csdl;
@@ -45,8 +45,8 @@ public static async Task<int> TransformOpenApiDocument(
             string? version,
             OpenApiFormat? format,
             LogLevel loglevel,
-            bool inline,
-            bool resolveexternal,
+            bool inlineLocal,
+            bool inlineExternal,
             string filterbyoperationids,
             string filterbytags,
             string filterbycollection,
@@ -111,8 +111,9 @@ CancellationToken cancellationToken
                         stopwatch.Restart();
                         var result = await new OpenApiStreamReader(new OpenApiReaderSettings
                         {
-                            ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences,
-                            RuleSet = ValidationRuleSet.GetDefaultRuleSet()
+                            RuleSet = ValidationRuleSet.GetDefaultRuleSet(),
+                            LoadExternalRefs = inlineExternal,
+                            BaseUrl = openapi.StartsWith("http") ? new Uri(openapi) : new Uri("file:" + new FileInfo(openapi).DirectoryName + "\\")
                         }
                         ).ReadAsync(stream);
 
@@ -189,7 +190,8 @@ CancellationToken cancellationToken
 
                     var settings = new OpenApiWriterSettings()
                     {
-                        ReferenceInline = inline ? ReferenceInlineSetting.InlineLocalReferences : ReferenceInlineSetting.DoNotInlineReferences
+                        InlineLocalReferences = inlineLocal,
+                        InlineExternalReferences = inlineExternal
                     };
 
                     IOpenApiWriter writer = openApiFormat switch
@@ -276,7 +278,7 @@ public static async Task<int> ValidateOpenApiDocument(
                         RuleSet = ValidationRuleSet.GetDefaultRuleSet()
                     }
                     ).ReadAsync(stream);
-
+                    
                     logger.LogTrace("{timestamp}ms: Completed parsing.", stopwatch.ElapsedMilliseconds);
 
                     document = result.OpenApiDocument;
@@ -353,6 +355,73 @@ public static async Task<OpenApiDocument> ConvertCsdlToOpenApi(Stream csdl)
             return document;
         }
 
+        /// <summary>
+        /// Fixes the references in the resulting OpenApiDocument.
+        /// </summary>
+        /// <param name="document"> The converted OpenApiDocument.</param>
+        /// <returns> A valid OpenApiDocument instance.</returns>
+        public static OpenApiDocument FixReferences(OpenApiDocument document)
+        {
+            // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance.
+            // So we write it out, and read it back in again to fix it up.
+
+            var sb = new StringBuilder();
+            document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb)));
+            var doc = new OpenApiStringReader().Read(sb.ToString(), out _);
+
+            return doc;
+        }
+
+        private static async Task<Stream> GetStream(string input, ILogger logger)
+        {
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            Stream stream;
+            if (input.StartsWith("http"))
+            {
+                try
+                {
+                    var httpClientHandler = new HttpClientHandler()
+                    {
+                        SslProtocols = System.Security.Authentication.SslProtocols.Tls12,
+                    };
+                    using var httpClient = new HttpClient(httpClientHandler)
+                    {
+                      DefaultRequestVersion = HttpVersion.Version20
+                    };
+                    stream = await httpClient.GetStreamAsync(input);
+                }
+                catch (HttpRequestException ex)
+                {
+                    logger.LogError($"Could not download the file at {input}, reason{ex}");
+                    return null;
+                }
+            }
+            else 
+            {
+                try
+                {
+                    var fileInput = new FileInfo(input);
+                    stream = fileInput.OpenRead();
+                }
+                catch (Exception ex) when (ex is FileNotFoundException ||
+                    ex is PathTooLongException ||
+                    ex is DirectoryNotFoundException ||
+                    ex is IOException ||
+                    ex is UnauthorizedAccessException ||
+                    ex is SecurityException ||
+                    ex is NotSupportedException)
+                {
+                    logger.LogError($"Could not open the file at {input}, reason: {ex.Message}");
+                    return null;
+                }
+            }
+            stopwatch.Stop();
+            logger.LogTrace("{timestamp}ms: Read file {input}", stopwatch.ElapsedMilliseconds, input);
+            return stream;
+        }
+
         /// <summary>
         /// Takes in a file stream, parses the stream into a JsonDocument and gets a list of paths and Http methods
         /// </summary>
@@ -365,42 +434,76 @@ public static Dictionary<string, List<string>> ParseJsonCollectionFile(Stream st
             logger.LogTrace("Parsing the json collection file into a JsonDocument");
             using var document = JsonDocument.Parse(stream);
             var root = document.RootElement;
-            var itemElement = root.GetProperty("item");
-            foreach (var requestObject in itemElement.EnumerateArray().Select(item => item.GetProperty("request")))
-            {
-                // Fetch list of methods and urls from collection, store them in a dictionary
-                var path = requestObject.GetProperty("url").GetProperty("raw").ToString();
-                var method = requestObject.GetProperty("method").ToString();
 
-                if (!requestUrls.ContainsKey(path))
+            requestUrls = EnumerateJsonDocument(root, requestUrls);
+            logger.LogTrace("Finished fetching the list of paths and Http methods defined in the Postman collection.");
+
+            return requestUrls;
+        }
+
+        private static Dictionary<string, List<string>> EnumerateJsonDocument(JsonElement itemElement, Dictionary<string, List<string>> paths)
+        {
+            var itemsArray = itemElement.GetProperty("item");
+             
+            foreach (var item in itemsArray.EnumerateArray())
+            {
+                if(item.ValueKind == JsonValueKind.Object)
                 {
-                    requestUrls.Add(path, new List<string> { method });
+                   if(item.TryGetProperty("request", out var request))
+                   {
+                        // Fetch list of methods and urls from collection, store them in a dictionary
+                        var path = request.GetProperty("url").GetProperty("raw").ToString();
+                        var method = request.GetProperty("method").ToString();
+                        if (!paths.ContainsKey(path))
+                        {
+                            paths.Add(path, new List<string> { method });
+                        }
+                        else
+                        {
+                            paths[path].Add(method);
+                        }
+                   }
+                   else
+                   {
+                        EnumerateJsonDocument(item, paths);
+                   }
                 }
                 else
                 {
-                    requestUrls[path].Add(method);
+                    EnumerateJsonDocument(item, paths);
                 }
             }
-            logger.LogTrace("Finished fetching the list of paths and Http methods defined in the Postman collection.");
-            return requestUrls;
+
+            return paths;
         }
 
         /// <summary>
         /// Fixes the references in the resulting OpenApiDocument.
         /// </summary>
         /// <param name="document"> The converted OpenApiDocument.</param>
         /// <returns> A valid OpenApiDocument instance.</returns>
-        private static OpenApiDocument FixReferences(OpenApiDocument document)
-        {
-            // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance.
-            // So we write it out, and read it back in again to fix it up.
-
-            var sb = new StringBuilder();
-            document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb)));
-            var doc = new OpenApiStringReader().Read(sb.ToString(), out _);
-
-            return doc;
-        }
+        // private static OpenApiDocument FixReferences2(OpenApiDocument document)
+        // {
+        //     // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance.
+        //     // So we write it out, and read it back in again to fix it up.
+
+        //     OpenApiDocument document;
+        //     logger.LogTrace("Parsing the OpenApi file");
+        //     var result = await new OpenApiStreamReader(new OpenApiReaderSettings
+        //     {
+        //         RuleSet = ValidationRuleSet.GetDefaultRuleSet(),
+        //         BaseUrl = new Uri(openapi)
+        //     }
+        //     ).ReadAsync(stream);
+
+        //     document = result.OpenApiDocument;
+        //     var context = result.OpenApiDiagnostic;
+        //     var sb = new StringBuilder();
+        //     document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb)));
+        //     var doc = new OpenApiStringReader().Read(sb.ToString(), out _);
+
+        //     return doc;
+        // }
 
         /// <summary>
         /// Reads stream from file system or makes HTTP request depending on the input string

src/Microsoft.OpenApi.Hidi/Program.cs

@@ -51,11 +51,11 @@ static async Task Main(string[] args)
             var filterByCollectionOption = new Option<string>("--filter-by-collection", "Filters OpenApiDocument by Postman collection provided");
             filterByCollectionOption.AddAlias("-c");
 
-            var inlineOption = new Option<bool>("--inline", "Inline $ref instances");
-            inlineOption.AddAlias("-i");
+            var inlineLocalOption = new Option<bool>("--inlineLocal", "Inline local $ref instances");
+            inlineLocalOption.AddAlias("-il");
 
-            var resolveExternalOption = new Option<bool>("--resolve-external", "Resolve external $refs");
-            resolveExternalOption.AddAlias("-ex");
+            var inlineExternalOption = new Option<bool>("--inlineExternal", "Inline external $ref instances");
+            inlineExternalOption.AddAlias("-ie");
 
             var validateCommand = new Command("validate")
             {
@@ -78,12 +78,12 @@ static async Task Main(string[] args)
                 filterByOperationIdsOption,
                 filterByTagsOption,
                 filterByCollectionOption,
-                inlineOption,
-                resolveExternalOption,
+                inlineLocalOption,
+                inlineExternalOption
             };
 
             transformCommand.SetHandler<string, string, string, FileInfo, bool, string?, OpenApiFormat?, LogLevel, bool, bool, string, string, string, CancellationToken> (
-                OpenApiService.TransformOpenApiDocument, descriptionOption, csdlOption, csdlFilterOption, outputOption, cleanOutputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption);
+                OpenApiService.TransformOpenApiDocument, descriptionOption, csdlOption, csdlFilterOption, outputOption, cleanOutputOption, versionOption, formatOption, logLevelOption, inlineLocalOption, inlineExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption);
 
             rootCommand.Add(transformCommand);
             rootCommand.Add(validateCommand);

src/Microsoft.OpenApi.Readers/OpenApiReaderSettings.cs

@@ -4,15 +4,10 @@
 using Microsoft.OpenApi.Any;
 using Microsoft.OpenApi.Interfaces;
 using Microsoft.OpenApi.Readers.Interface;
-using Microsoft.OpenApi.Readers.ParseNodes;
-using Microsoft.OpenApi.Readers.Services;
 using Microsoft.OpenApi.Validations;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Microsoft.OpenApi.Readers
 {
@@ -30,7 +25,7 @@ public enum ReferenceResolutionSetting
         /// </summary>
         ResolveLocalReferences,
         /// <summary>
-        /// Convert all references to references of valid domain objects.
+        /// ResolveAllReferences effectively means load external references. Will be removed in v2. External references are never "resolved".
         /// </summary>
         ResolveAllReferences
     }
@@ -43,8 +38,14 @@ public class OpenApiReaderSettings
         /// <summary>
         /// Indicates how references in the source document should be handled.
         /// </summary>
+        /// <remarks>This setting will be going away in the next major version of this library.  Use GetEffective on model objects to get resolved references.</remarks>
         public ReferenceResolutionSetting ReferenceResolution { get; set; } = ReferenceResolutionSetting.ResolveLocalReferences;
 
+        /// <summary>
+        /// When external references are found, load them into a shared workspace
+        /// </summary>
+        public bool LoadExternalRefs { get; set; } = false;
+
         /// <summary>
         /// Dictionary of parsers for converting extensions into strongly typed classes
         /// </summary>

src/Microsoft.OpenApi.Readers/OpenApiStreamReader.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.OpenApi.Interfaces;
@@ -23,6 +24,12 @@ public class OpenApiStreamReader : IOpenApiReader<Stream, OpenApiDiagnostic>
         public OpenApiStreamReader(OpenApiReaderSettings settings = null)
         {
             _settings = settings ?? new OpenApiReaderSettings();
+
+            if((_settings.ReferenceResolution == ReferenceResolutionSetting.ResolveAllReferences || _settings.LoadExternalRefs)
+                && _settings.BaseUrl == null)
+            {
+                throw new ArgumentException("BaseUrl must be provided to resolve external references.");
+            }
         }
 
         /// <summary>