Add --imports switch and improve exported assets copying (#980)

* Add --imports switch.
Copy whole sub-directory of ExportedAssets including npm.

* Always put in ImportedAssets

* Ensure uischema keys are sorted.

Co-authored-by: Emilio Munoz <[email protected]>

Author: Chris McConnell

packages/dialog/README.md

@@ -36,7 +36,7 @@ _See code: [src/commands/dialog/index.ts](https://github.com/microsoft/botframew
 
 ## `bf dialog:merge PATTERNS`
 
-Merge `<kind>.schema` and `<kind>[.<locale>].uischema` definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  For C#, ensures all nuget declarative resources in ExportedAssets are copied to ImportedAssets in the same location.
+Merge `<kind>.schema` and `<kind>[.<locale>].uischema` definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  If a dependent package has an ExportedAssets directory it is copied to ImportedAssets/<package> in the --imports directory.
 
 ```
 USAGE
@@ -52,6 +52,9 @@ OPTIONS
   -v, --verbose          Show verbose logging of files as they are processed.
   --extension=extension  [default: .dialog,.lg,.lu,.schema,.qna,.uischema] Extension to include as a resource for C#.
 
+  --imports=imports      Output path for imported assets.  Defaults to the directory of --out with an ImportedAssets
+                         directory.
+
 EXAMPLES
   $ bf dialog:merge myProject.csproj plugins/*.nuspec
   $ bf dialog:merge package.json -o app.schema

packages/dialog/src/commands/dialog/merge.ts

@@ -7,7 +7,7 @@ import {Command, flags} from '@microsoft/bf-cli-command'
 import SchemaMerger from '../../library/schemaMerger'
 
 export default class DialogMerge extends Command {
-    static description = 'Merge `<kind>.schema` and `<kind>[.<locale>].uischema` definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  For C#, ensures all nuget declarative resources in ExportedAssets are copied to ImportedAssets in the same location.'
+    static description = 'Merge `<kind>.schema` and `<kind>[.<locale>].uischema` definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  If a dependent package has an ExportedAssets directory it is copied to ImportedAssets/<package> in the --imports directory.'
 
     static args = [
         {name: 'patterns', required: true, description: 'Any number of glob regex patterns to match .csproj, .nuspec or package.json files.'},
@@ -20,6 +20,7 @@ export default class DialogMerge extends Command {
         extension: flags.string({description: 'Extension to include as a resource for C#.', required: false, multiple: true, default: ['.dialog', '.lg', '.lu', '.schema', '.qna', '.uischema']}),
         help: flags.help({char: 'h'}),
         nugetRoot: flags.string({description: 'Nuget root directory for debugging.', hidden: true}),
+        imports: flags.string({description: 'Output path for imported assets.  Defaults to the directory of --out with an ImportedAssets directory.', required: false}),
         output: flags.string({char: 'o', description: 'Output path and filename for merged .schema and .uischema.  Defaults to first project name.', required: false}),
         schema: flags.string({char: 's', description: 'Path to merged .schema file to use if merging .uischema only.', required: false}),
         verbose: flags.boolean({char: 'v', description: 'Show verbose logging of files as they are processed.', default: false}),
@@ -32,7 +33,7 @@ export default class DialogMerge extends Command {
 
     async run() {
         const {argv, flags} = this.parse(DialogMerge)
-        let merger = new SchemaMerger(argv, flags.output, flags.verbose, this.log, this.warn, this.error, flags.extension, flags.schema, flags.debug, flags.nugetRoot)
+        let merger = new SchemaMerger(argv, flags.output, flags.imports, flags.verbose, this.log, this.warn, this.error, flags.extension, flags.schema, flags.debug, flags.nugetRoot)
         await merger.merge()
     }
 }

packages/dialog/src/library/schemaMerger.ts

@@ -60,7 +60,7 @@ function normalize(path: string): string {
 
 // Deep merge of JSON objects
 function mergeObjects(obj1: any, obj2: any): any {
-    let target = {};
+    let target = {}
     let merger = (obj: any) => {
         for (let prop in obj) {
             let val = obj[prop]
@@ -156,6 +156,11 @@ class Component {
         return patterns
     }
 
+    // Test to see if root component
+    public isRoot(): boolean {
+        return this.parents.length === 0 || (this.parents.length === 1 && this.parents[0].parents.length === 0)
+    }
+
     // Check to see if this component is a parent of another component
     public isParent(node: Component): boolean {
         let found = false
@@ -201,6 +206,7 @@ export default class SchemaMerger {
     // Input parameters
     private readonly patterns: string[]
     private output: string
+    private readonly imports: string
     private readonly verbose: boolean
     private readonly log: any
     private readonly warn: any
@@ -257,6 +263,7 @@ export default class SchemaMerger {
      * Merger to combine component .schema and .uischema files to make a custom schema.
      * @param patterns Glob patterns for the .csproj or packge.json files to combine.
      * @param output The output file to create or empty string to use default.
+     * @param imports The output directory for imports.
      * @param verbose True to show files as processed.
      * @param log Logger for informational messages.
      * @param warn Logger for warning messages.
@@ -266,9 +273,10 @@ export default class SchemaMerger {
      * @param debug Generate debug output.
      * @param nugetRoot Root directory for nuget packages.  (Useful for testing.)
      */
-    public constructor(patterns: string[], output: string, verbose: boolean, log: any, warn: any, error: any, extensions?: string[], schema?: string, debug?: boolean, nugetRoot?: string) {
+    public constructor(patterns: string[], output: string, imports: string | undefined, verbose: boolean, log: any, warn: any, error: any, extensions?: string[], schema?: string, debug?: boolean, nugetRoot?: string) {
         this.patterns = patterns
         this.output = output ? ppath.join(ppath.dirname(output), ppath.basename(output, ppath.extname(output))) : ''
+        this.imports = imports ?? ppath.dirname(output)
         this.verbose = verbose
         this.log = log
         this.warn = warn
@@ -565,7 +573,10 @@ export default class SchemaMerger {
             }
             if (!this.failed) {
                 for (let locale of Object.keys(result)) {
-                    let uischema = {$schema: this.metaUISchemaId, ...result[locale]}
+                    let uischema = {$schema: this.metaUISchemaId}
+                    for (let key of Object.keys(result[locale]).sort()) {
+                        uischema[key] = result[locale][key]
+                    }
                     this.currentFile = ppath.join(ppath.dirname(this.output), outputName + (locale ? '.' + locale : '') + '.uischema')
                     this.log(`Writing ${this.currentFile}`)
                     await fs.writeJSON(this.currentFile, uischema, this.jsonOptions)
@@ -587,39 +598,17 @@ export default class SchemaMerger {
         return [kindName, locale]
     }
 
-    // For C# copy all assets into generated/<package>/
+    // Copy all exported assets into imported assets
     private async copyAssets(): Promise<void> {
-        if (!this.failed && !this.schemaPath) {
-            let isCS = false
+        if (!this.failed && !this.schemaPath && this.components.length > 0) {
+            this.log(`Copying exported assets to ${this.imports}`)
             for (let component of this.components) {
-                if (component.path.endsWith('.csproj') || component.path.endsWith('.nuspec')) {
-                    isCS = true
-                    break
-                }
-            }
-            if (isCS) {
-                let generatedPath = ppath.join(ppath.dirname(this.output), 'ImportedAssets')
-                let found = false
-                for (let files of this.files.values()) {
-                    for (let componentPaths of files.values()) {
-                        for (let componentPath of componentPaths) {
-                            let component = componentPath.component
-                            let path = componentPath.path
-                            let relativePath = ppath.relative(ppath.dirname(component.path), path)
-                            // Copy anything found in exportedassets outside of project
-                            if (!component.isCSProject() && relativePath.toLowerCase().startsWith('exportedassets')) {
-                                // Copy package files to output
-                                if (!found) {
-                                    found = true
-                                    this.log(`Copying C# package exported assets to ${generatedPath}`)
-                                }
-                                let remaining = relativePath.substring('exportedAssets/'.length)
-                                let outputPath = ppath.join(generatedPath, componentPath.component.name, remaining)
-                                this.vlog(`Copying ${path} to ${outputPath}`)
-                                await fs.ensureDir(ppath.dirname(outputPath))
-                                await fs.copyFile(path, outputPath)
-                            }
-                        }
+                if (!component.isRoot()) {
+                    let exported = ppath.join(ppath.dirname(component.path), 'exportedAssets')
+                    if (await fs.pathExists(exported)) {
+                        let imported = ppath.join(this.imports, 'ImportedAssets', component.name)
+                        this.vlog(`Copying ${exported} to ${imported}`)
+                        await fs.copy(exported, imported, {recursive: true})
                     }
                 }
             }
@@ -718,24 +707,6 @@ export default class SchemaMerger {
         })
     }
 
-    // Resolver for schema: -> metaSchema
-    private schemaProtocolResolver(): any {
-        let reader = (_file: parser.FileInfo) => {
-            return JSON.stringify(this.metaSchema)
-        }
-        return {
-            resolve: {
-                defintion: {
-                    order: 1,
-                    canRead: /^schema:/i,
-                    read(file: parser.FileInfo, _callback: any, _$refs: any) {
-                        return reader(file)
-                    }
-                }
-            }
-        }
-    }
-
     private indent(): string {
         return '  '.repeat(this.parents.length)
     }
@@ -917,7 +888,7 @@ export default class SchemaMerger {
                             let dependentPath = ppath.join(rootDir, 'node_modules', dependent, 'package.json')
                             if (await fs.pathExists(dependentPath)) {
                                 await this.expandPackageJson(dependentPath)
-                                break;
+                                break
                             } else {
                                 rootDir = ppath.dirname(rootDir)
                             }

packages/dialog/test/commands/dialog/merge.test.ts

@@ -32,6 +32,7 @@ async function merge(patterns: string[], output?: string, verbose?: boolean, sch
     }
     let merger = new SchemaMerger(patterns,
         output ? ppath.join(tempDir, output) : '',
+        undefined,        
         verbose || false,
         logger, logger, logger,
         undefined, 
@@ -168,10 +169,8 @@ describe('dialog:merge', async () => {
         assert(countMatches(/Following.*project3/, lines) === 1, 'Did not follow project1')
         assert(countMatches(/Following nuget.*nuget3.*1.0.0/, lines) === 1, 'Did not follow nuget3')
         assert(countMatches(/Parsing.*nuget3.component1.schema/, lines) === 1, 'Missing nuget3.component1.schema')
-        assert(countMatches(/Copying/i, lines) === 4, 'Wrong number of copies')
-        assert(countMatches(/Copying.*nuget3.lg/i, lines) === 1, 'Did not copy .lg')
-        assert(countMatches(/Copying.*nuget3.lu/i, lines) === 1, 'Did not copy .lu')
-        assert(countMatches(/Copying.*nuget3.qna/i, lines) === 1, 'Did not copy .qna')
+        assert(countMatches(/Copying/i, lines) === 2, 'Wrong number of copies')
+        assert(countMatches(/Copying.*nuget3/i, lines) === 1, 'Did not copy nuget3')
         assert(await fs.pathExists(ppath.join(tempDir, 'ImportedAssets', 'nuget3', 'stuff', 'nuget3.qna')), 'Did not copy directory')
         await compareToOracle('project3.schema')
         await compareToOracle('project3.en-us.uischema')
@@ -224,6 +223,9 @@ describe('dialog:merge', async () => {
         assert(countMatches('dependent-package.schema', lines) === 1, 'Missing dependent-package.schema')
         assert(countMatches('parent-package.schema', lines) === 1, 'Missing parent-package.schema')
         assert(countMatches('no-package.schema', lines) === 0, 'Extra no-package.schema')
+        assert(countMatches('Copying', lines) === 2, 'Wrong number of copies')
+        assert(await fs.pathExists(ppath.join(tempDir, 'ImportedAssets', 'dependent-package', 'assets', 'dependent-package.jpg')), 'Incomplete assets copy')
+        assert(!await fs.pathExists(ppath.join(tempDir, 'ImportedAssets', 'root-package')), 'Copied rootx')
         await compareToOracle('root-package.schema')
         await compareToOracle('root-package.uischema')
     })