Adding MLPerfExecutor creating directory instead of mkdir (#324)

* Adding MLPerfExecutor creating directory

* Bumping up version

* adding unit test fix

---------

Co-authored-by: Deepanshu Vaid <[email protected]>

Author: deep1712

.pipelines/azure-pipelines-linux.yml

@@ -16,7 +16,7 @@ resources:
     options: --entrypoint=""
 
 variables:
-  VcVersion : 1.14.31
+  VcVersion : 1.14.32
   ROOT: $(Build.SourcesDirectory)
   CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
   ENABLE_PRS_DELAYSIGN: 1

.pipelines/azure-pipelines.yml

@@ -18,7 +18,7 @@ pool:
   vmImage: windows-latest
 
 variables:
-  VcVersion : 1.14.31
+  VcVersion : 1.14.32
   ROOT: $(Build.SourcesDirectory)
   CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
   ENABLE_PRS_DELAYSIGN: 1

src/VirtualClient/VirtualClient.Actions.UnitTests/MLPerf/MLPerfExecutorTests.cs

@@ -151,8 +151,6 @@ public async Task MLPerfExecutorInitializesWorkloadAsExpected()
             List<string> expectedCommands = new List<string>
             {
                 "sudo usermod -aG docker anyuser",
-                "sudo bash -c \"export MLPERF_SCRATCH_PATH=/dev/sdd1/scratch" +
-                " && mkdir $MLPERF_SCRATCH_PATH/data $MLPERF_SCRATCH_PATH/models $MLPERF_SCRATCH_PATH/preprocessed_data\"",
                 "sudo -u anyuser bash -c \"make prebuild MLPERF_SCRATCH_PATH=/dev/sdd1/scratch\"",
                 "sudo docker ps",
                 "sudo docker exec -u anyuser mlperf-inference-anyuser-x86_64 sudo bash -c " +
@@ -196,6 +194,18 @@ public async Task MLPerfExecutorInitializesWorkloadAsExpected()
             })
             .ReturnsAsync(makeFileString);
 
+            IEnumerable<string> expectedDirectories = new List<string>
+            {
+                "/dev/sdd1/scratch/data",
+                "/dev/sdd1/scratch/models",
+                "/dev/sdd1/scratch/preprocessed_data"
+            };
+
+            this.mockFixture.Directory.Setup(d => d.CreateDirectory(It.IsAny<string>())).Callback<string>((directory) =>
+            {
+                Assert.IsTrue(expectedDirectories.Select(ed => directory.Contains(ed)).Any());
+            });
+
             List<string> commandsExecuted = new List<string>();
             this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) =>
             {
@@ -246,6 +256,17 @@ public async Task MLPerfExecutorExecutesAsExpected()
                 .ReturnsAsync(makeFileString);
 
             IEnumerable<string> expectedCommands = this.GetExpectedCommands();
+            IEnumerable<string> expectedDirectories = new List<string>
+            {
+                "/dev/sdd1/scratch/data",
+                "/dev/sdd1/scratch/models",
+                "/dev/sdd1/scratch/preprocessed_data"
+            };
+
+            this.mockFixture.Directory.Setup(d => d.CreateDirectory(It.IsAny<string>())).Callback<string>((directory) =>
+            {
+                Assert.IsTrue(expectedDirectories.Select(ed => directory.Contains(ed)).Any());
+            });
 
             List<string> commandsExecuted = new List<string>();
 
@@ -298,8 +319,6 @@ private IEnumerable<string> GetExpectedCommands()
             commands = new List<string>
             {
                 "sudo usermod -aG docker anyuser",
-                "sudo bash -c \"export MLPERF_SCRATCH_PATH=/dev/sdd1/scratch" +
-                " && mkdir $MLPERF_SCRATCH_PATH/data $MLPERF_SCRATCH_PATH/models $MLPERF_SCRATCH_PATH/preprocessed_data\"",
                 "sudo -u anyuser bash -c \"make prebuild MLPERF_SCRATCH_PATH=/dev/sdd1/scratch\"",
                 "sudo docker ps",
                 "sudo docker exec -u anyuser mlperf-inference-anyuser-x86_64 sudo bash -c " +

src/VirtualClient/VirtualClient.Actions/MLPerf/MLPerfExecutor.cs

@@ -332,11 +332,9 @@ protected async Task SetupEnvironmentAsync(CancellationToken cancellationToken)
         {
             string dockerExecCommand = $"docker exec -u {this.Username} {this.GetContainerName()}";
 
-            await this.ExecuteCommandAsync(
-                "bash", 
-                $"-c \"{this.ExportScratchSpace} && mkdir $MLPERF_SCRATCH_PATH/data $MLPERF_SCRATCH_PATH/models $MLPERF_SCRATCH_PATH/preprocessed_data\"", 
-                this.NvidiaDirectory, 
-                cancellationToken);
+            this.fileSystem.Directory.CreateDirectory(this.PlatformSpecifics.Combine(this.mlperfScratchSpace, "data"));
+            this.fileSystem.Directory.CreateDirectory(this.PlatformSpecifics.Combine(this.mlperfScratchSpace, "models"));
+            this.fileSystem.Directory.CreateDirectory(this.PlatformSpecifics.Combine(this.mlperfScratchSpace, "preprocessed_data"));
 
             await this.ExecuteCommandAsync(
                 "sudo",