Allow for building helix payloads directly from build artifacts in python (#4911)

* Add test manifest support for running microbenchmarks

* Add missing argument to script

* Fix bugs

* Fix command line parsing

* Abstract out payload building

* Minor fixes

* Abstracting out more parts

* Some more cleanup/fixes

* Remove csproj changes (kept in separate branch)

* Add some documentation/cleanup

* Address PR comments

Author: caaavik-msft

.gitignore

@@ -12,11 +12,16 @@
 **/.pytest_cache/
 
 ## VSCode .gitignore (included from https://github.com/github/gitignore/blob/master/Global/VisualStudioCode.gitignore
-.vscode/
+.vscode/*
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
+!.vscode/*.code-snippets
+!*.code-workspace
+
+# Built Visual Studio Code Extensions
+*.vsix
 
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.

.vscode/settings.json

@@ -10,14 +10,6 @@
         "**/packages/": true,
         "/tools/": true,
     },
-    "python.analysis.typeCheckingMode": "strict",
-    "python.analysis.diagnosticSeverityOverrides": {
-        "reportMissingParameterType": "none",
-        "reportUnnecessaryComparison": "information",
-        "reportUnnecessaryIsInstance": "information",
-        "reportUnusedVariable": "information",
-        "reportMissingTypeStubs": "information"
-    },
-    "python.analysis.extraPaths": ["scripts"],
-    "python.analysis.diagnosticMode": "workspace"
+    "python.analysis.typeCheckingMode": "standard",
+    "python.analysis.extraPaths": ["scripts"]
 }

eng/pipelines/templates/run-performance-job.yml

@@ -56,6 +56,7 @@ parameters:
   iOSLlvmBuild: false               # optional -- Whether to build iOS with LLVM
   iOSStripSymbols: false            # optional -- Whether to strip symbols from the iOS build
   additionalSetupParameters: ''     # optional -- Additional arguments to pass to the script
+  liveLibrariesBuildConfig: ''      # optional -- Build configuration when generating Core_Root for libraries
 
 jobs:
 - template: ${{ parameters.jobTemplate }}
@@ -226,6 +227,10 @@ jobs:
               - '--internal'
               - ${{ if eq(parameters.downloadPdn, true) }}:
                 - '--pdn-path "$(_pdnPath)"'
+            - ${{ if ne(parameters.buildConfig, '') }}:
+              - '--build-config ${{ parameters.buildConfig }}'
+            - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+              - '--live-libraries-build-config ${{ parameters.liveLibrariesBuildConfig }}'
             - ${{ if ne(parameters.additionalSetupParameters, '') }}:
               - '${{ parameters.additionalSetupParameters }}'
       - template: /eng/pipelines/templates/send-to-helix-step.yml

eng/pipelines/templates/runtime-perf-job.yml

@@ -16,7 +16,6 @@ parameters:
   hybridGlobalization: 'False'
   isScenario: false
   downloadSpecificBuild: null # buildId, pipeline, branchName, project
-  crossBuild: false
   runtimeRepoAlias: runtime
   performanceRepoAlias: self
   selfIsRuntime: true
@@ -37,7 +36,7 @@ jobs:
     # Test job depends on the corresponding build job
     ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
       dependsOn:
-        - ${{ if not(or(in(parameters.runtimeType, 'AndroidMono', 'AndroidCoreCLR', 'iOSMono', 'iOSNativeAOT', 'wasm'), and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')))) }}:
+        - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'AndroidCoreCLR', 'iOSMono', 'iOSNativeAOT', 'wasm', 'mono')) }}:
           - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'coreclr') }}
         - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}:
           - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'mono') }}
@@ -56,20 +55,10 @@ jobs:
           - ${{ 'build_ios_arm64_release_iOSNativeAOT' }}
 
     variables:
-      - name: _crossBuildPropertyArg
-        value: ''
-      - ${{ if eq(parameters.crossBuild, true) }}:
-        - name: _crossBuildPropertyArg
-          value: '-cross'
       - name: librariesDownloadDir
         value: $(Build.SourcesDirectory)/artifacts
       - name: builtAppDir
         value: $(Build.SourcesDirectory)
-      - name: librariesOverrideArg
-        value: ''
-      - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
-        - name: librariesOverrideArg
-          value: '/p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}'
 
       - name: buildConfigUpper
         ${{ if eq(parameters.buildConfig, 'debug') }}:
@@ -82,13 +71,9 @@ jobs:
       - ${{ if eq(parameters.osGroup, 'windows') }}:
         - name: archiveExtension
           value: '.zip'
-        - name: scriptExt
-          value: '.cmd'
       - ${{ else }}:
         - name: archiveExtension
           value: '.tar.gz'
-        - name: scriptExt
-          value: '.sh'
 
       - ${{ each variable in parameters.variables }}:
         - ${{ variable }}
@@ -119,34 +104,13 @@ jobs:
               artifactName: BrowserWasm
               displayName: BrowserWasm
               ${{ insert }}: ${{ parameters.downloadSpecificBuild }}
-
-        - script: >-
-            mkdir -p $(librariesDownloadDir)/bin/wasm/wasm-data &&
-            mkdir -p $(librariesDownloadDir)/bin/wasm/dotnet &&
-            cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest/* $(librariesDownloadDir)/bin/wasm/dotnet &&
-            cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm &&
-            cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/wasm-data/test-main.js &&
-            find $(librariesDownloadDir)/bin/wasm -type d &&
-            find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \;
-          displayName: "Create wasm directory (Linux)"
       - ${{ elseif and(eq(parameters.codeGenType, 'AOT'), not(eq(parameters.runtimeType, 'AndroidMono'))) }}:
-        # Download mono AOT
-        - template: /eng/pipelines/templates/download-artifact-step.yml
-          parameters:
-            unpackFolder: $(librariesDownloadDir)/bin
-            artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr$(archiveExtension)'
-            artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr'
-            displayName: 'Runtime artifacts'
-
         - template: /eng/pipelines/templates/download-artifact-step.yml
           parameters:
             unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT
             artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz
             artifactName: LinuxMonoAOT${{ parameters.archType }}
             displayName: AOT Mono Artifacts
-
-        - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack"
-          displayName: "Create aot directory (Linux)"
       - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}:
         - template: /eng/pipelines/templates/download-artifact-step.yml
           parameters:
@@ -276,32 +240,4 @@ jobs:
               artifactName: 'iOSNativeAOTArm64NoStripSymbolsBuildLog'
             ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}:
               artifactName: 'iOSNativeAOTArm64StripSymbolsBuildLog'
-            checkDownloadedFiles: true
-
-      - ${{ if notIn(parameters.runtimeType, 'wasm', 'AndroidMono', 'AndroidCoreCLR', 'iOSMono', 'iOSNativeAOT') }}:
-        - ${{ if ne(parameters.runtimeFlavor, 'Mono') }}:
-          # Create Core_Root
-          - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg)
-            displayName: Create Core_Root
-        - ${{ elseif ne(parameters.osGroup, 'windows') }}:
-          - task: CopyFiles@2
-            displayName: Create .dotnet-mono folder
-            inputs:
-              SourceFolder: $(librariesDownloadDir)/bin/testhost/${{ parameters.framework }}-${{ parameters.osGroup }}-$(buildConfigUpper)-${{ parameters.archType }}
-              TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono
-          - task: CopyFiles@2
-            displayName: Copy corerun to .dotnet-mono
-            inputs:
-              SourceFolder: $(librariesDownloadDir)/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)
-              Contents: 'corerun*'
-              TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion)
-
-        - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm')) }}:
-          - script: >-
-              mkdir -p $(librariesDownloadDir)/bin/wasm/data &&
-              cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm &&
-              cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm &&
-              cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js &&
-              find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \;
-            displayName: "Create wasm directory (Linux)"
-            condition: succeeded()
+            checkDownloadedFiles: true

requirements.txt

@@ -4,3 +4,5 @@ azure.identity==1.16.1
 dataclasses==0.8
 gitpython<=3.1.41
 urllib3==1.26.19
+opentelemetry-api==1.23.0
+opentelemetry-sdk==1.23.0

scripts/benchmarks_ci.py

@@ -87,42 +87,6 @@ def add_arguments(parser: ArgumentParser) -> ArgumentParser:
     # Restore/Build/Run functionality for MicroBenchmarks.csproj
     micro_benchmarks.add_arguments(parser)
 
-    PRODUCT_INFO = [
-        'init-tools',  # Default
-        'repo',
-        'cli',
-        'args',
-    ]
-    parser.add_argument(
-        '--cli-source-info',
-        dest='cli_source_info',
-        required=False,
-        default=PRODUCT_INFO[0],
-        choices=PRODUCT_INFO,
-        help='Specifies where the product information comes from.',
-    )
-    parser.add_argument(
-        '--cli-branch',
-        dest='cli_branch',
-        required=False,
-        type=str,
-        help='Product branch.'
-    )
-    parser.add_argument(
-        '--cli-commit-sha',
-        dest='cli_commit_sha',
-        required=False,
-        type=str,
-        help='Product commit sha.'
-    )
-    parser.add_argument(
-        '--cli-repository',
-        dest='cli_repository',
-        required=False,
-        type=str,
-        help='Product repository.'
-    )
-
     def __is_valid_dotnet_path(dp: str) -> str:
         if not os.path.isdir(dp):
             raise ArgumentTypeError('Path {} does not exist'.format(dp))
@@ -142,24 +106,6 @@ def __is_valid_dotnet_path(dp: str) -> str:
         help='Path to a custom dotnet'
     )
 
-    def __is_valid_datetime(dt: str) -> str:
-        try:
-            datetime.strptime(dt, '%Y-%m-%dT%H:%M:%SZ')
-            return dt
-        except ValueError:
-            raise ArgumentTypeError(
-                'Datetime "{}" is in the wrong format.'.format(dt))
-
-    parser.add_argument(
-        '--cli-source-timestamp',
-        dest='cli_source_timestamp',
-        required=False,
-        type=__is_valid_datetime,
-        help='''Product timestamp of the soruces used to generate this build
-            (date-time from RFC 3339, Section 5.6.
-            "%%Y-%%m-%%dT%%H:%%M:%%SZ").'''
-    )
-
     parser.add_argument('--upload-to-perflab-container',
         dest="upload_to_perflab_container",
         required=False,