Switch to axios to support proxy server when merging. (#1190)

* Switch to axios to support proxy server when merging.
Normalize sorting to reduce variation.

* modify https proxy setting

* refactor

* fix lint

* fix lint

Co-authored-by: Chris McConnell <chrimc>
Co-authored-by: cosmicshuai <[email protected]>

Author: Chris McConnell

.vscode/launch.json

@@ -157,6 +157,26 @@
         "./packages/lu/lib/**"
       ]
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Dialog merge root-package schema",
+      "preLaunchTask": "${defaultBuildTask}",
+      "program": "${workspaceFolder}/packages/dialog/bin/run",
+      "outputCapture": "std",
+      "outFiles": [
+        "./packages/dialog/lib/**"
+      ],
+      "args": [
+        "dialog:merge",
+        "npm/node_modules/root-package/package.json",
+        "-o",
+        "oracles/root-package",
+        "--verbose"
+      ],
+      "internalConsoleOptions": "openOnSessionStart",
+      "cwd": "${workspaceFolder}/packages/dialog/test/commands/dialog/"
+    },
     {
       "type": "node",
       "request": "launch",

common/config/rush/pnpm-lock.yaml

@@ -45,10 +45,11 @@
     "@types/lru-cache": "^5.1.0",
     "@types/xml2js": "^0.4.4",
     "ajv": "^6.12.2",
+    "axios": "~0.21.1",
+    "https-proxy-agent": "^5.0.0",
     "chalk": "^2.4.2",
     "clone": "^2.1.2",
     "fs-extra": "^8.1.0",
-    "get-uri": "~3.0.2",
     "globby": "^11.0.0",
     "@apidevtools/json-schema-ref-parser": "^9.0.1",
     "os": "~0.1.1",
@@ -80,4 +81,4 @@
     "tslint": "^5.18.0",
     "typescript": "^4.0.3"
   }
-}
+}

packages/dialog/package.json

@@ -0,0 +1,51 @@
+/*!
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from 'fs-extra'
+const axios = require('axios')
+const url = require('url')
+const httpsProxyAgent = require('https-proxy-agent')
+
+const httpsProxy = function (config) {
+  const parsed = url.parse(config.url)
+  const protocol = parsed.protocol
+  if (protocol !== 'https:') {
+      return config
+  }
+
+  /* tslint:disable:no-string-literal */
+  const envProxy = process.env['HTTPS_PROXY'] || process.env['https_proxy']
+  /* tslint:enable:no-string-literal */
+  if (envProxy) {
+    const parsed = url.parse(envProxy)
+    const proxyOpt = {
+    hostname: parsed.hostname,
+    port: parsed.port
+    }
+
+    if (parsed.auth) {
+    (proxyOpt as any).auth = parsed.auth
+    }
+    
+    config.httpsAgent = httpsProxyAgent(proxyOpt)
+    //Disable direct proxy
+    config.proxy = false
+  }
+
+  return config
+}
+axios.interceptors.request.use(httpsProxy)
+
+const filePrefix = 'file:///'
+
+// Get JSON from a URL.
+export default async function getJSON(url: string): Promise<any> {
+  if (url.startsWith(filePrefix)) {
+    return fs.readJSON(url.substring(filePrefix.length))
+  }
+
+  const resp = await axios.get(url);
+  return resp.data
+}

packages/dialog/src/library/getJSON.ts

@@ -12,10 +12,10 @@ import * as xp from 'xml2js'
 import Ajv = require('ajv')
 import parser from '@apidevtools/json-schema-ref-parser'
 import {JsonPointer as ptr} from 'json-ptr'
+import getJSON from './getJSON'
 
 const allof = require('json-schema-merge-allof')
 const clone = require('clone')
-const getUri = require('get-uri')
 const glob = require('globby')
 const semverRsort = require('semver/functions/rsort')
 const util = require('util')
@@ -40,16 +40,6 @@ function pathName(path: string | undefined, extension: string): string {
     return path ? `${path}/${extension}` : extension
 }
 
-// Get JSON from a URI.
-async function getJSON(uri: string): Promise<any> {
-    const stream = await getUri(uri)
-    let data = ''
-    for await (const chunk of stream) {
-        data += chunk.toString()
-    }
-    return JSON.parse(data)
-}
-
 // Convert to the right kind of slash. 
 // ppath.normalize did not do this properly on the mac.
 function normalize(path: string): string {
@@ -67,6 +57,11 @@ function forwardSlashes(input: string): string {
     return input.replace(/\\/g, '/')
 }
 
+// Consistent sort comparison
+function sortFunction(a: any, b: any): number {
+    return a.localeCompare(b, 'en')
+}
+
 // Deep merge of JSON objects
 function mergeObjects(obj1: any, obj2: any): any {
     const target = {}
@@ -83,7 +78,7 @@ function mergeObjects(obj1: any, obj2: any): any {
     merger(obj1)
     merger(obj2)
     const finalTarget = {}
-    for (const key of Object.keys(target).sort()) {
+    for (const key of Object.keys(target).sort(sortFunction)) {
         finalTarget[key] = target[key]
     }
     return finalTarget
@@ -649,7 +644,7 @@ export class SchemaMerger {
         this.validateAndExpandPolicies()
         const oneOf = Object.keys(this.definitions)
             .filter(kind => !this.isInterface(kind) && this.definitions[kind].$role)
-            .sort()
+            .sort(sortFunction)
             .map(kind => {
                 return {$ref: `#/definitions/${kind}`}
             })
@@ -661,7 +656,7 @@ export class SchemaMerger {
             this.currentFile = this.output + '.schema'
             this.currentKind = ''
             const finalDefinitions: any = {}
-            for (const key of Object.keys(this.definitions).sort()) {
+            for (const key of Object.keys(this.definitions).sort(sortFunction)) {
                 finalDefinitions[key] = this.definitions[key]
             }
             let finalSchema: any = {
@@ -778,7 +773,7 @@ export class SchemaMerger {
             if (!this.failed) {
                 for (const locale of Object.keys(result)) {
                     const uischema = {$schema: this.metaUISchemaId}
-                    for (const key of Object.keys(result[locale]).sort()) {
+                    for (const key of Object.keys(result[locale]).sort(sortFunction)) {
                         uischema[key] = result[locale][key]
                     }
                     this.currentFile = ppath.join(ppath.dirname(this.output), outputName + (locale ? '.' + locale : '') + '.uischema')
@@ -1788,7 +1783,7 @@ export class SchemaMerger {
         for (this.currentKind in this.definitions) {
             const definition = this.definitions[this.currentKind]
             if (this.isInterface(this.currentKind) && definition.oneOf) {
-                definition.oneOf = definition.oneOf.sort((a: any, b: any) => (a.$ref || a.type).localeCompare(b.$ref || b.type))
+                definition.oneOf = definition.oneOf.sort((a: any, b: any) => sortFunction(a.$ref ?? a.type, b.$ref ?? b.type))
             }
         }
     }

packages/dialog/src/library/schemaMerger.ts

@@ -5,20 +5,14 @@
 
 import Ajv = require('ajv')
 import parser from '@apidevtools/json-schema-ref-parser'
-
-let getUri: any = require('get-uri')
+import http from './getJSON'
 
 // Get JSON from a URI.
 async function getJSON(uri: string): Promise<any> {
     if (uri.indexOf(':') < 2) {
         uri = `file:///${uri}`
     }
-    let stream = await getUri(uri)
-    let data = ''
-    for await (let chunk of stream) {
-        data += chunk.toString()
-    }
-    return JSON.parse(data)
+    return http(uri)
 }
 
 export class SchemaTracker {

packages/dialog/src/library/schemaTracker.ts

@@ -148,7 +148,7 @@ describe('dialog:merge', async () => {
         console.log('Start missing component schema')
         const [merged, lines] = await merge(['schemas/badSchemas/missingComponent.schema'], 'app.schema')
         assert(!merged, 'Merge should have failed')
-        assert(countMatches(/file does not exist/i, lines) === 1, 'No missing component schema')
+        assert(countMatches(/status code 404/i, lines) === 1, 'No missing component schema')
     })
 
     it('mismatched component schema', async () => {
@@ -495,7 +495,7 @@ describe('dialog:merge', async () => {
             .persist()
         const [merged, lines] = await merge(['schemas/*.schema'], 'app.schema')
         assert(!merged, 'Merging should fail')
-        assert(countMatches(/internal server error/i, lines) === 1, 'Did not detect server error')
+        assert(countMatches(/status code 500/i, lines) === 1, 'Did not detect server error')
         scope.done()
         nock.cleanAll()
     })