Refactor file upload descriptor factory to handle full content path template formats using metadata and component parameter entries for resolving placeholders.

Author: brdeyo

.pipelines/azure-pipelines-linux.yml

@@ -9,66 +9,104 @@ trigger:
 
 pr: none
 
-pool:
-  vmImage: ubuntu-latest
+resources:
+  containers:
+  - container: pmc-cli
+    image: 'mcr.microsoft.com/pmc/pmc-cli:latest'
+    options: --entrypoint=""
 
 variables:
-  VcVersion : 1.10.5
+  VcVersion : 1.11.1
   ROOT: $(Build.SourcesDirectory)
   CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
   ENABLE_PRS_DELAYSIGN: 1
   GitVersion.SemVer: ''
-  
 
-steps:
+stages:
+- stage: BuildPackage
+  jobs:
+  - job: BuildPackage
+    pool:
+      vmImage: 'ubuntu-latest'
 
-- task: PipAuthenticate@1
-  inputs:
-    artifactFeeds: 'CRC-Dev'
-- script: pip install pmc-cli
-- task: UseDotNet@2
-  continueOnError: false
-  inputs:
-    packageType: 'sdk'
-    version: '6.0.406'
-    useGlobalJson: false
-    performMultiLevelLookup: true
-    workingDirectory: '$(Build.SourcesDirectory)'
+    steps:
+    - task: UseDotNet@2
+      continueOnError: false
+      inputs:
+        packageType: 'sdk'
+        version: '6.0.406'
+        useGlobalJson: false
+        performMultiLevelLookup: true
+        workingDirectory: '$(Build.SourcesDirectory)'
 
-- script: chmod -R +x $(Build.SourcesDirectory)
-  displayName: 'Allow scripts to be executable on Linux'
+    - script: chmod -R +x $(Build.SourcesDirectory)
+      displayName: 'Allow scripts to be executable on Linux'
 
-  # Ensure the artifact output/bin/obj directories are clean.
-# - script: $(Build.SourcesDirectory)/clean.cmd
-  # displayName: 'Clean Output Directories'
-  
-  # Build the repo.
-- script: $(Build.SourcesDirectory)/build.sh $(VcVersion)
-  displayName: 'Build Solutions'
+      # Ensure the artifact output/bin/obj directories are clean.
+    # - script: $(Build.SourcesDirectory)/clean.cmd
+      # displayName: 'Clean Output Directories'
+      
+      # Build the repo.
+    - script: $(Build.SourcesDirectory)/build.sh $(VcVersion) --build-all
+      displayName: 'Build Solutions'
 
-  # Build NuGet packages for the services/agents in the repo.
-# - script: $(Build.SourcesDirectory)/build-packages.cmd $(VcVersion)$(VersionSuffix)
-  # displayName: 'Build NuGet Packages'
+    - script: $(Build.SourcesDirectory)/src/VirtualClient/VirtualClient.Packaging/build-deb-linux-x64.sh $(VcVersion)
+      displayName: 'Build deb package for linux x64'
 
-- task: EsrpCodeSigning@1
-  inputs:
-    ConnectedServiceName: 'virtualclient-esrp'
-    FolderPath: '$(System.DefaultWorkingDirectory)'
-    Pattern: '*.*'
-    signConfigType: 'inlineSignParams'
-    inlineOperation: |
-      [
-          {
-              "KeyCode" : "CP-450779-Pgp",
-              "OperationCode" : "LinuxSign",
-              "Parameters" : {},
-              "ToolName" : "sign",
-              "ToolVersion" : "1.0"
-          }
-      ]
-    SessionTimeout: '60'
-    MaxConcurrency: '50'
-    MaxRetryAttempts: '5'
+    - script: $(Build.SourcesDirectory)/src/VirtualClient/VirtualClient.Packaging/build-deb-linux-arm64.sh $(VcVersion)
+      displayName: 'Build deb package for linux arm64'
 
-#- script: $(Build.SourcesDirectory)/upload-packages.cmd $(Build.SourcesDirectory)/out/packages $(NUGETORGAPIKEY)
-#  displayName: 'Publish NuGet Packages'
+    - task: EsrpCodeSigning@1
+      inputs:
+        ConnectedServiceName: 'virtualclient-esrp'
+        FolderPath: '$(Build.SourcesDirectory)/out/packages'
+        Pattern: '*.deb'
+        signConfigType: 'inlineSignParams'
+        inlineOperation: |
+          [
+              {
+                  "KeyCode" : "CP-450779-Pgp",
+                  "OperationCode" : "LinuxSign",
+                  "Parameters" : {},
+                  "ToolName" : "sign",
+                  "ToolVersion" : "1.0"
+              }
+          ]
+        SessionTimeout: '60'
+        MaxConcurrency: '50'
+        MaxRetryAttempts: '5'
+        
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathToPublish: '$(Build.SourcesDirectory)/out/packages'
+        artifactName: drop
+
+
+  - job: PublishPackage
+    condition: false
+    dependsOn: BuildPackage
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: pmc-cli
+
+    steps:
+    - task: PipAuthenticate@1
+      inputs:
+        artifactFeeds: 'CRC-Dev'
+    - script: pip install pmc-cli
+        
+    - task: AzureKeyVault@2
+      inputs:
+        ConnectedServiceName: 'VirtualClient-AzureRM-AME'
+        KeyVaultName: 'junoprod01vault01'
+        SecretsFilter: 'virtualclient-cert-pem'
+        RunAsPreJob: true
+
+    - task: PublishLinuxPackagesPMC@0
+      inputs:
+        profile: 'ppe' # or tux-dev or ppe or prod depending on what you selected
+        msal_SNIAuth: 'msal-sniauth'
+        msal_cert: $(virtualclient-cert-pem)
+        msal_client_ID: 'f407a268-1184-4845-92b1-eebfbe0749c0'
+        package_path: '$(Build.SourcesDirectory)/out/packages/*.deb'
+        repository: 'virtualclient-apt'

.pipelines/azure-pipelines.yml

@@ -18,7 +18,7 @@ pool:
   vmImage: windows-latest
 
 variables:
-  VcVersion : 1.10.5
+  VcVersion : 1.11.1
   ROOT: $(Build.SourcesDirectory)
   CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
   ENABLE_PRS_DELAYSIGN: 1

build.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+if [ -n "$1" ]; then
+    VCBuildVersion="$1"
+fi
+
+if [ -z "$VCBuildVersion" ]; then
+    VCBuildVersion="0.0.1"
+fi
+
 while [[ "$#" -gt 0 ]]; do
     case $1 in
         --build-all)

src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs

@@ -106,7 +106,7 @@ private IEnumerable<string> GetProfileExpectedCommands(PlatformID platform)
                 case PlatformID.Win32NT:
                     commands = new List<string>
                     {
-                        @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net6.0",
+                        @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net7.0",
                         @"dotnet\.exe .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive",
                         @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json"
                     };
@@ -116,7 +116,7 @@ private IEnumerable<string> GetProfileExpectedCommands(PlatformID platform)
                     commands = new List<string>
                     {
                         @"chmod \+x .+bombardier",
-                        @"dotnet build -c Release -p:BenchmarksTargetFramework=net6.0",
+                        @"dotnet build -c Release -p:BenchmarksTargetFramework=net7.0",
                         @"dotnet .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive",
                         @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json"
                     };

src/VirtualClient/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj

@@ -23,7 +23,7 @@
         <Content Include="..\TestResources\oltp-c.fio.jobfile" Link="Resources\oltp-c.fio.jobfile">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         </Content>
-        <Content Include="..\TestResources\OpenFOAMResults.txt" Link="Resources\OpenFOAMResults.txt">
+        <Content Include="..\TestResources\OpenFoamResults.txt" Link="Resources\OpenFoamResults.txt">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         </Content>
         <Content Include="..\TestResources\Pbzip2Results.txt" Link="Resources\Pbzip2Results.txt">

src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/SPECview/resultCSV_multipleViewSets.csv

@@ -0,0 +1,89 @@
+Composites
+3dsmax-07,35.46
+catia-06,43.3
+creo-03,56.98
+energy-03,27.46
+maya-06,227.8
+medical-03,35.16
+snx-04,139.11
+solidworks-07,97.8
+
+viewset,index,name,weight,fps
+3dsmax-07,1,3dsmax_Arch_Shaded,9.52,50.87
+3dsmax-07,2,3dsmax_Arch_Graphite,9.52,42.35
+3dsmax-07,3,3dsmax_Space_Wireframe,9.52,147.76
+3dsmax-07,4,3dsmax_Space_Clay,9.52,110.5
+3dsmax-07,5,3dsmax_ProjectSoane_Shaded,9.52,7.91
+3dsmax-07,6,3dsmax_ProjectSoane_Wireframe,9.52,9.82
+3dsmax-07,7,3dsmax_HugeFish_Wireframe,9.52,11.39
+3dsmax-07,8,3dsmax_HugeFish_Shaded,9.52,11.54
+3dsmax-07,9,3dsmax_Office_Realistic,9.52,72.91
+3dsmax-07,10,3dsmax_Office_Shaded,9.52,106.39
+3dsmax-07,11,3dsmax_Office_RealisticMats,4.8,45.2
+catia-06,1,catiaV5test1,14.28,34.01
+catia-06,2,catiaV5test2,14.28,44.3
+catia-06,3,catiaV5test3,0,424.59
+catia-06,4,catiaV5test4,14.28,158.01
+catia-06,5,CATIA_3DX_repEngine,14.29,15.86
+catia-06,6,CATIA_3DX_loftJet,14.29,26.73
+catia-06,7,CATIA_3DX_car,14.29,46.68
+catia-06,8,CATIA_3DX_Multi_car,14.29,60.68
+creo-03,1,Creo_03_test_01,8.33,60.44
+creo-03,2,Creo_03_test_02,8.33,25.21
+creo-03,3,Creo_03_test_03,8.34,33.31
+creo-03,4,Creo_03_test_04,10,172.31
+creo-03,5,Creo_03_test_05,10,244.89
+creo-03,6,Creo_03_test_06,10,72.06
+creo-03,7,Creo_03_test_07,10,40.56
+creo-03,8,Creo_03_test_08,10,50.14
+creo-03,9,Creo_03_test_09,10,12.34
+creo-03,10,Creo_03_test_10,3.75,92.99
+creo-03,11,Creo_03_test_11,3.75,56.1
+creo-03,12,Creo_03_test_12,3.75,49.47
+creo-03,13,Creo_03_test_13,3.75,85.21
+energy-03,1,Test1,16.67,38.02
+energy-03,2,Test2,16.67,20.46
+energy-03,3,Test3,16.67,18.04
+energy-03,4,Test4,16.67,49.49
+energy-03,5,Test5,16.66,25.79
+energy-03,6,Test6,16.66,23.92
+maya-06,1,Maya_01,8.33,120.22
+maya-06,2,Maya_02,8.33,664.73
+maya-06,3,Maya_03,12.5,106.89
+maya-06,4,Maya_04,12.5,459.27
+maya-06,5,Maya_05,12.5,245.47
+maya-06,6,Maya_06,8.33,86.95
+maya-06,7,Maya_07,8.33,198.37
+maya-06,8,Maya_08,12.5,388.78
+maya-06,9,Maya_09,8.34,506.82
+maya-06,10,Maya_10,8.34,87.18
+medical-03,1,Test1,10,258.34
+medical-03,2,Test2,10,310.64
+medical-03,3,Test3,10,63.19
+medical-03,4,Test4,10,26.93
+medical-03,5,Test5,10,32.06
+medical-03,6,Test6,10,82.55
+medical-03,7,Test7,10,31.91
+medical-03,8,Test8,10,53.17
+medical-03,9,Test9,10,1.26
+medical-03,10,Test10,10,3.74
+snx-04,1,NX8_AdvancedStudioAA,7.5,123.54
+snx-04,2,NX8_ShadedAA,10,164.72
+snx-04,3,NX8_ShadedWithEdgeAA,20,85.33
+snx-04,4,NX8_StudioAA,5,114.15
+snx-04,5,NX8_powerTrain_Wireframe,7.5,158.74
+snx-04,6,NX8_suvAdvancedStudio,7.5,238.02
+snx-04,7,NX8_suvShaded,10,216.75
+snx-04,8,NX8_suvShadedWithEdge,20,118.66
+snx-04,9,NX8_suvStudio2,5,134.47
+snx-04,10,NX8_suvWireframe,7.5,232.72
+solidworks-07,1,SW2020_RallyCar_Realview,10,139.26
+solidworks-07,2,SW2020_RallyCar_ShadedEdges,10,191.75
+solidworks-07,3,SW2020_RallyCar_Shaded,15,256.76
+solidworks-07,4,SW2020_Excavator_Shaded,5,268.97
+solidworks-07,5,SW2020_Excavator_Realview,15,146.99
+solidworks-07,6,SW2020_JetEngine_ShadedEdges,10,105.26
+solidworks-07,7,SW2020_JetEngine_Shaded,10,167.8
+solidworks-07,8,SW2020_JetEngine_Realview,10,92.75
+solidworks-07,9,SW2020_SpaceCrawler_Shaded,5,9.3
+solidworks-07,10,SW2020_SpaceCrawler_ShadedEdges,10,4.99

src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/SPECview/resultCSV_oneViewSet.csv

@@ -0,0 +1,15 @@
+Composites
+3dsmax-07,29.33
+
+viewset,index,name,weight,fps
+3dsmax-07,1,3dsmax_Arch_Shaded,9.52,45.9
+3dsmax-07,2,3dsmax_Arch_Graphite,9.52,38.25
+3dsmax-07,3,3dsmax_Space_Wireframe,9.52,101.46
+3dsmax-07,4,3dsmax_Space_Clay,9.52,95.22
+3dsmax-07,5,3dsmax_ProjectSoane_Shaded,9.52,8.3
+3dsmax-07,6,3dsmax_ProjectSoane_Wireframe,9.52,10.3
+3dsmax-07,7,3dsmax_HugeFish_Wireframe,9.52,6.46
+3dsmax-07,8,3dsmax_HugeFish_Shaded,9.52,7.26
+3dsmax-07,9,3dsmax_Office_Realistic,9.52,63.75
+3dsmax-07,10,3dsmax_Office_Shaded,9.52,92.39
+3dsmax-07,11,3dsmax_Office_RealisticMats,4.8,40.56