Support merging .uischema files (#852)

* Topo sort in progress.

* Update sort.

* Working sort

* Breadth first sort.

* Move to building component tree.

* Normalize slash in tests.

* Update documentation

* Add --nugetRoot switch.

* Add $package to app.schema.

* Intermediate progress to switch machines.

* Update to move machines.

* Update to offical URLS.
Upgrade json-ptr to 1.3
Full uischema test examples.

* Oracles and working tests.

* All csproj tests working

* All tests work.

* Fix packages.config.

* Add copying C# asset files into output

* Renable test

* Order uischema keys and update oracles.

* Move write outside of loop.

* Add missing .uischema files.

* Use / in paths

* Update documentation.

* Update docs.

Co-authored-by: Tom Laird-McConnell <[email protected]>

Author: Chris McConnell

.vscode/launch.json

@@ -34,7 +34,7 @@
                 "999999",
                 "--colors",
                 "-g",
-                "dialog:merge.*"
+                ".*"
             ],
             "internalConsoleOptions": "openOnSessionStart",
             "outputCapture": "std",
@@ -195,8 +195,7 @@
                 "dialog:merge",
                 "schemas/*.schema",
                 "-o",
-                "schemas/app.schema",
-                "-d"
+                "schemas/app.schema"
             ],
             "internalConsoleOptions": "openOnSessionStart",
             "cwd": "${workspaceFolder}/packages/dialog/test/commands/dialog"
@@ -242,6 +241,75 @@
             "internalConsoleOptions": "openOnSessionStart",
             "cwd": "${env:TEMP}/sandwich.out"
         },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "LUIS Build Simple",
+            "preLaunchTask": "${defaultBuildTask}",
+            "program": "${workspaceFolder}/packages/luis/bin/run",
+            "outputCapture": "std",
+            "outFiles": [
+                "./packages/luis/lib/**",
+                "./packages/lu/lib/**"
+            ],
+            "args": [
+                "luis:build",
+                "--luConfig",
+                "luconfig.json",
+                "--authoringKey",
+                "${env:LUIS_AUTHORING_KEY}"
+            ],
+            "internalConsoleOptions": "openOnSessionStart",
+            "cwd": "${env:TEMP}/simple.out"
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "LUIS Convert Sandwich",
+            "preLaunchTask": "${defaultBuildTask}",
+            "program": "${workspaceFolder}/packages/luis/bin/run",
+            "outputCapture": "std",
+            "outFiles": [
+                "./packages/luis/lib/**",
+                "./packages/lu/lib/**"
+            ],
+            "args": [
+                "luis:convert",
+                "--in",
+                "en-us/sandwich.en-us.lu",
+                "--name",
+                "sandwich",
+                "-o",
+                "converted.json",
+                "--force"
+            ],
+            "internalConsoleOptions": "openOnSessionStart",
+            "cwd": "${env:TEMP}/sandwich.out"
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "LUIS Convert Simple",
+            "preLaunchTask": "${defaultBuildTask}",
+            "program": "${workspaceFolder}/packages/luis/bin/run",
+            "outputCapture": "std",
+            "outFiles": [
+                "./packages/luis/lib/**",
+                "./packages/lu/lib/**"
+            ],
+            "args": [
+                "luis:convert",
+                "--in",
+                "en-us/simple.en-us.lu",
+                "--name",
+                "simple",
+                "-o",
+                "converted.json",
+                "--force"
+            ],
+            "internalConsoleOptions": "openOnSessionStart",
+            "cwd": "${env:TEMP}/simple.out"
+        },
         {
             "type": "node",
             "request": "launch",

common/config/rush/pnpm-lock.yaml

@@ -13,9 +13,12 @@ This package is intended for Microsoft use only and should be consumed through @
 
 # Commands
 <!-- commands -->
-* [`bf dialog`](#bf-dialog)
-* [`bf dialog:merge PATTERNS`](#bf-dialogmerge-patterns)
-* [`bf dialog:verify PATTERNS`](#bf-dialogverify-patterns)
+- [@microsoft/bf-dialog](#microsoftbf-dialog)
+- [Relevant docs](#relevant-docs)
+- [Commands](#commands)
+  - [`bf dialog`](#bf-dialog)
+  - [`bf dialog:merge PATTERNS`](#bf-dialogmerge-patterns)
+  - [`bf dialog:verify PATTERNS`](#bf-dialogverify-patterns)
 
 ## `bf dialog`
 
@@ -33,23 +36,24 @@ _See code: [src/commands/dialog/index.ts](https://github.com/microsoft/botframew
 
 ## `bf dialog:merge PATTERNS`
 
-Merge component .schema files into an app.schema.
+Merge <kind>.schema and <kind>[.<locale>].uischema definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  For C#, ensures all nuget declarative resources are included in the same location.
 
 ```
 USAGE
   $ bf dialog:merge PATTERNS
 
 ARGUMENTS
-  PATTERNS  Any number of glob regex patterns to match .schema, .csproj, or package.json files.
+  PATTERNS  Any number of glob regex patterns to match .csproj, .nuspec or package.json files.
 
 OPTIONS
-  -h, --help           show CLI help
-  -o, --output=output  [default: app.schema] Output path and filename for merged schema.
-  -v, --verbose        Show verbose logging of files as they are processed.
+  -h, --help             show CLI help
+  -o, --output=output    Output path and filename for merged .schema and .uischema.  Defaults to first project name.
+  -v, --verbose          Show verbose logging of files as they are processed.
+  --extension=extension  [default: .dialog,.lg,.lu,.schema,.qna,.uischema] Extension to include as a resource for C#.
 
 EXAMPLES
-  $ bf dialog:merge *.csproj
-  $ bf dialog:merge libraries/**/*.schema **/*.csproj -o app.schema
+  $ bf dialog:merge myProject.csproj plugins/*.nuspec
+  $ bf dialog:merge package.json -o app.schema
 ```
 
 _See code: [src/commands/dialog/merge.ts](https://github.com/microsoft/botframework-cli/tree/master/packages/dialog/src/commands/dialog/merge.ts)_

packages/dialog/README.md

@@ -2,18 +2,30 @@
 
 ## Merge
 
-This will merge together [Microsoft Bot Builder](https://github.com/Microsoft/BotBuilder) .schema JSON schema files into a single JSON schema file. You can point to the files either directly with a glob pattern or indirectly through a glob pattern that matches a package.json, packages.config or \*.csproj file. The .schema files should have a unique filename that is used to refer to that type using `$kind`. The .schema files should include a `$schema: "https://raw.githubusercontent.com/microsoft/botframework-sdk/master/schemas/component/component.schema"` which defines the schema they are validated against. You can access common definitions by using `$ref: "schema:#/definitions/stringExpression"` in a property definition.  At the top-level in a component schema you can use `$role:"implements(<kind>)` to project the type definition into interface types defined using `$role:"interface"` while merging. To extend an existing .schema file use `$role: "extends(<kind>)"`.  To refer to a type in a property, use `"$kind":"<kind>"` and the corresponding kind will be wired into your schema.  The merger combines all of the component .schema files into a single .schema file that has resolved all external `$ref`, merged `allOf` and connected together schemas through `$role` and `$kind`.
+This will merge together [Microsoft Bot Builder](https://github.com/Microsoft/BotBuilder) .schema JSON schema files into a single JSON schema file and .uischema files into a per-locale .uischema file.  You can point to the files either directly with a glob pattern or indirectly through a glob pattern that matches a package.json or /*.csproj file. The .schema files should have a unique filename that is used to refer to that type using `$kind`. The .schema files should include a `$schema: "https://schemas.botframework.com/schemas/component/v1.0/component.schema"` which defines the schema they are validated against. 
 
-You can also mark properties with a `$role: "expression"` to indicate to tooling that an expression is allowed for that property. 
+You can also mark properties with a `$role: "expression"` to indicate to tooling that an expression is allowed for that property. A simple way to do this is to make use of common expression definitions like `$ref: "schema:#/definitions/<type>Expression"` in a property definition.  
 
-For example look at these files:
+At the top-level in a component schema you can use `$role:"implements(<kind>)` to project the type definition into interface types defined using `$role:"interface"` while merging. 
 
-- [IRecognizer.schema](test/schemas/IRecognizer.schema) defines the place holder for `IRecognizer` including a default option which is a bare string.
-- [Recognizer.schema](test/schemas/Recognizer.schema) includes `$role:"implements(IRecognizer)"` which extends the `IRecognizer` definition when merged.
-- [root.schema](test/schemas/root.schema) is a schema file that includes `$kind:"IRecognizer"` in order to make use of the `IRecognizer` place holder.  The `$role: []` ensures that this is available as a top-level object.
-- [app.schema](test/schemas/app.schema) was created by this tool shows how all of these definitions are merged together. In particular if you look at `IRecognizer` you will see the definition that includes a string, or the complete definition of `Recognizer`.
+To extend an existing .schema file use `$role: "extends(<kind>)"`.  This will add onto the existing definitions in `<kind>`.
 
-[root.dialog](test/examples/root.dialog) Shows how you could use the resulting schema to enter in JSON schema and get intellisense.
+To refer to a type in a property, use `$kind:"<kind>"` and the corresponding kind will be wired into your schema.  The merger combines all of the component .schema files into a single .schema file that has resolved all external `$ref`, merged `allOf` and connected together schemas through `$role` and `$kind`.
+
+For `<kind>[.<locale>].uischema` files, use `$schema: "https://schemas.botframework.com/schemas/ui/v1.0/ui.schema"` to define their contents.  The `<kind>` must be found in the .schema file and the optional `<locale>` is a string like en-us.  All of the individual .uischema files will be combined into a composite one per-locale.  If definitions conflict, the definition found first in the topological sort of bread-first, parent before child will be used. 
+
+For C#, nuget does not deal well with content files so all declarative .dialog, .lu, .lg, and .qna files will be copied into `generated/<package>` so you can easily include all of them in your project output.  
+
+For examples look at these files:
+
+- [IRecognizer.schema](../test/schemas/IRecognizer.schema) defines the place holder for `IRecognizer` including a default option which is a bare string.
+- [Recognizer.schema](../test/schemas/Recognizer.schema) includes `$role:"implements(IRecognizer)"` which extends the `IRecognizer` definition when merged.
+- [root.schema](../test/schemas/root.schema) is a schema file that includes `$kind:"IRecognizer"` in order to make use of the `IRecognizer` place holder.  The `$role: []` ensures that this is available as a top-level object.
+- [app.schema](../test/schemas/app.schema) was created by this tool shows how all of these definitions are merged together. In particular if you look at `IRecognizer` you will see the definition that includes a string, or the complete definition of `Recognizer`.
+- [nuget3.en-us.uischema](../test/commands/dialog/projects/project3/nuget3.en-us.uischema) shows an example component .uischema file.
+- [project3.en-us.uischema](../test/commands/dialog/oracles/project3.en-us.uischema) shows a merged .uischema file.
+
+[root.dialog](../test/examples/root.dialog) Shows how you could use the resulting schema to enter in JSON schema and get intellisense.
 
 ## Verify
 

packages/dialog/docs/readme.md

@@ -55,7 +55,7 @@
     "seedrandom": "~3.0.5",
     "tslib": "^1.10.0",
     "xml2js": "^0.4.19",
-    "json-ptr": "~1.2.0",
+    "json-ptr": "~1.3.0",
     "json-schema-merge-allof": "~0.7.0",
     "@snyk/nuget-semver": "~1.3.0"
   },

packages/dialog/package.json

@@ -3,33 +3,35 @@
  * Licensed under the MIT License.
  */
 
-import { Command, flags } from '@microsoft/bf-cli-command'
+import {Command, flags} from '@microsoft/bf-cli-command'
 import SchemaMerger from '../../library/schemaMerger'
 
 export default class DialogMerge extends Command {
-    static description = 'Merge component .schema files into an app.schema.'
+    static description = 'Merge <kind>.schema and <kind>[.<locale>].uischema definitions from a project and its dependencies into a single .schema for describing .dialog files and a per locale .uischema for describing how Composer shows them.  For C#, ensures all nuget declarative resources are included in the same location.'
 
     static args = [
-        { name: 'patterns', required: true, description: 'Any number of glob regex patterns to match .schema, .csproj, or package.json files.'},
+        {name: 'patterns', required: true, description: 'Any number of glob regex patterns to match .csproj, .nuspec or package.json files.'},
     ]
 
     static strict = false
 
     static flags: flags.Input<any> = {
-        debug: flags.boolean({ char: 'd', description: 'Generate debug files.', hidden: true, default: false}),
-        help: flags.help({ char: 'h' }),
-        output: flags.string({ char: 'o', description: 'Output path and filename for merged schema.', default: 'app.schema', required: false }),
-        verbose: flags.boolean({ char: 'v', description: 'Show verbose logging of files as they are processed.', default: false }),
+        debug: flags.boolean({char: 'd', description: 'Generate debug files.', hidden: true, default: false}),
+        extension: flags.string({description: 'Extension to include as a resource for C#.', required: false, multiple: true, default: ['.dialog', '.lg', '.lu', '.schema', '.qna', '.uischema']}),
+        help: flags.help({char: 'h'}),
+        nugetRoot: flags.string({description: 'Nuget root directory for debugging.', hidden: true}),
+        output: flags.string({char: 'o', description: 'Output path and filename for merged .schema and .uischema.  Defaults to first project name.', required: false}),
+        verbose: flags.boolean({char: 'v', description: 'Show verbose logging of files as they are processed.', default: false}),
     }
 
     static examples = [
-        '$ bf dialog:merge *.csproj',
-        '$ bf dialog:merge libraries/**/*.schema **/*.csproj -o app.schema'
+        '$ bf dialog:merge myProject.csproj plugins/*.nuspec',
+        '$ bf dialog:merge package.json -o app.schema'
     ]
 
     async run() {
-        const { argv, flags } = this.parse(DialogMerge)
-        let merger = new SchemaMerger(argv, flags.output, flags.verbose, this.log, this.warn, this.error, flags.debug)
-        await merger.mergeSchemas()
+        const {argv, flags} = this.parse(DialogMerge)
+        let merger = new SchemaMerger(argv, flags.output, flags.verbose, this.log, this.warn, this.error, flags.extension, flags.debug, flags.nugetRoot)
+        await merger.merge()
     }
 }