Update sample projects to support variable slots scheduling (#293)

* Update sample projects to support variable slots scheduling

* Refactor task slots per task and add it into output

* Bump Microsoft.Azure.Batch package to 14.0.0

Author: lilinvictorms

CSharp/ArticleProjects/ApplicationInsights/TopNWords/TopNWords.csproj

@@ -10,11 +10,11 @@
     <ProjectReference Include="..\..\..\Common\Microsoft.Azure.Batch.Samples.Common.csproj" />
     <ProjectReference Include="..\Microsoft.Azure.Batch.Samples.TelemetryInitializer\Microsoft.Azure.Batch.Samples.TelemetryInitializer.csproj" />
     <ProjectReference Include="..\Microsoft.Azure.Batch.Samples.TelemetryStartTask\Microsoft.Azure.Batch.Samples.TelemetryStartTask.csproj" />
-    
+
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.FileStaging" Version="8.3.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.0" />

CSharp/ArticleProjects/EfficientListQueries/EfficientListQueries.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/EfficientListQueries/Program.cs

@@ -47,7 +47,7 @@ private static async Task MainAsync(string[] args)
             // You may adjust these values to experiment with different compute resource scenarios.
             const string nodeSize     = "standard_d1_v2";
             const int nodeCount       = 1;
-            const int maxTasksPerNode = 4;
+            const int taskSlotsPerNode = 4;
 
             // Adjust the task count to experiment with different list operation query durations
             const int taskCount = 5000;
@@ -72,8 +72,8 @@ private static async Task MainAsync(string[] args)
                     poolId,
                     nodeSize,
                     nodeCount,
-                    maxTasksPerNode);
-                
+                    taskSlotsPerNode);
+
                 // Create a CloudJob, or obtain an existing job with the specified ID
                 CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
 
@@ -190,9 +190,9 @@ private static async Task QueryTasksAsync(BatchClient batchClient, string jobId,
             Stopwatch stopwatch = Stopwatch.StartNew();
 
             taskList.AddRange(await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync());
-            
+
             stopwatch.Stop();
-            
+
             Console.WriteLine("{0} tasks retrieved in {1} (ExpandClause: {2} | FilterClause: {3} | SelectClause: {4})",
                 taskList.Count,
                 stopwatch.Elapsed,

CSharp/ArticleProjects/JobPrepRelease/JobPrepRelease.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/MultiInstanceTasks/MultiInstanceTasks.csproj

@@ -23,7 +23,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
   </ItemGroup>
 

CSharp/ArticleProjects/MultiInstanceTasks/Program.cs

@@ -39,7 +39,7 @@ public static void Main(string[] args)
                 Console.ReadLine();
             }
         }
-        
+
         public static async Task MainAsync()
         {
             const string poolId = "MultiInstanceSamplePool";
@@ -206,7 +206,7 @@ private static async Task CreatePoolAsync(BatchClient batchClient, string poolId
             // sample, MPIHelloWorld.exe) running on the different nodes
             unboundPool.InterComputeNodeCommunicationEnabled = true;
             // REQUIRED for multi-instance tasks
-            unboundPool.MaxTasksPerComputeNode = 1;
+            unboundPool.TaskSlotsPerNode = 1;
 
             // Specify the application and version to deploy to the compute nodes.
             unboundPool.ApplicationPackageReferences = new List<ApplicationPackageReference>

CSharp/ArticleProjects/ParallelTasks/ParallelTasks.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
   </ItemGroup>
 
 </Project>

CSharp/ArticleProjects/ParallelTasks/Program.cs

@@ -45,13 +45,13 @@ public static void Main(string[] args)
         private static async Task MainAsync(string[] args)
         {
             // You may adjust these values to experiment with different compute resource scenarios.
-            const string nodeSize     = "standard_d1_v2";
-            const int nodeCount       = 4;
-            const int maxTasksPerNode = 4;
-            const int taskCount       = 32;
+            const string nodeSize      = "standard_d1_v2";
+            const int nodeCount        = 4;
+            const int taskSlotsPerNode = 4;
+            const int taskCount        = 32;
 
             // Ensure there are enough tasks to help avoid hitting some timeout conditions below
-            int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
+            int minimumTaskCount = nodeCount * taskSlotsPerNode * 2;
             if (taskCount < minimumTaskCount)
             {
                 Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
@@ -61,6 +61,18 @@ private static async Task MainAsync(string[] args)
                 return;
             }
 
+            // In this sample, each task is assigned with random required slots; adjust these values
+            // to simulate variable task slots together with allowed task slots per compute node
+            const int maxTaskSlots     = 2;
+            if (maxTaskSlots > taskSlotsPerNode)
+            {
+                Console.WriteLine("Invalid task slot configuration: maxTaskSlots for task should not be greater than pool's TaskSlotsPerNode");
+                Console.WriteLine();
+
+                // Invalid task slot configuration, exit the application
+                return;
+            }
+
             // In this sample, the tasks simply ping localhost on the compute nodes; adjust these
             // values to simulate variable task duration
             const int minPings = 30;
@@ -89,7 +101,7 @@ private static async Task MainAsync(string[] args)
                     poolId,
                     nodeSize,
                     nodeCount,
-                    maxTasksPerNode);
+                    taskSlotsPerNode);
 
                 // Create a CloudJob, or obtain an existing pool with the specified ID
                 CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
@@ -102,7 +114,10 @@ private static async Task MainAsync(string[] args)
                 {
                     string taskId = "task" + i.ToString().PadLeft(3, '0');
                     string taskCommandLine = "ping -n " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
-                    CloudTask task = new CloudTask(taskId, taskCommandLine);
+                    CloudTask task = new CloudTask(taskId, taskCommandLine)
+                    {
+                        RequiredSlots = rand.Next(1, maxTaskSlots + 1)
+                    };
                     tasks.Add(task);
                 }
 
@@ -129,6 +144,18 @@ private static async Task MainAsync(string[] args)
                 await GettingStartedCommon.PrintNodeTasksAsync(batchClient, pool.Id);
                 Console.WriteLine();
 
+                // Print out running task and task slot counts on all nodes.
+                Console.WriteLine();
+                await GettingStartedCommon.PrintNodeTaskCountsAsync(batchClient, pool.Id);
+                Console.WriteLine();
+
+                // Print out task and task slot counts of the job.
+                // Note: it may have delay as Batch service is aggregating task counts for the job.
+                await Task.Delay(TimeSpan.FromSeconds(5));
+                Console.WriteLine();
+                await GettingStartedCommon.PrintJobTaskCountsAsync(batchClient, jobId);
+                Console.WriteLine();
+
                 // Pause execution while we wait for all of the tasks to complete
                 Console.WriteLine("Waiting for task completion...");
                 Console.WriteLine();
@@ -147,10 +174,16 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                 stopwatch.Stop();
 
+                // Print out job task counts again.
+                await Task.Delay(TimeSpan.FromSeconds(5));
+                Console.WriteLine();
+                await GettingStartedCommon.PrintJobTaskCountsAsync(batchClient, jobId);
+                Console.WriteLine();
+
                 // Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
                 // If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
                 // amount of data transferred, lowering your query response times in increasing performance.
-                ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
+                ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state,requiredSlots");
                 IPagedEnumerable<CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
 
                 // Get a collection of the completed tasks sorted by the compute nodes on which they executed
@@ -173,7 +206,7 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                     lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
 
-                    Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
+                    Console.WriteLine($"\t{task.Id} (slots={task.RequiredSlots}): {task.CommandLine}");
                 }
 
                 // Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
@@ -200,11 +233,12 @@ await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
 
                 // Print some summary information
                 Console.WriteLine();
-                Console.WriteLine("             Nodes: " + nodeCount);
-                Console.WriteLine("         Node size: " + nodeSize);
-                Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
-                Console.WriteLine("             Tasks: " + tasks.Count);
-                Console.WriteLine("          Duration: " + stopwatch.Elapsed);
+                Console.WriteLine("              Nodes: " + nodeCount);
+                Console.WriteLine("          Node size: " + nodeSize);
+                Console.WriteLine("Task slots per node: " + pool.TaskSlotsPerNode);
+                Console.WriteLine(" Max slots per task: " + maxTaskSlots);
+                Console.WriteLine("              Tasks: " + tasks.Count);
+                Console.WriteLine("           Duration: " + stopwatch.Elapsed);
                 Console.WriteLine();
                 Console.WriteLine("Done!");
                 Console.WriteLine();

CSharp/ArticleProjects/ParallelTasks/README.md

@@ -1,3 +1,7 @@
 ### ParallelTasks
 
-This console application sample project backs the code snippets found in [Maximize Azure Batch compute resource usage with concurrent node tasks](http://azure.microsoft.com/documentation/articles/batch-parallel-node-tasks/). The application demonstrates the creation of a Batch pool whose compute nodes are configured for executing multiple concurrent tasks, and prints node and task information to the console during execution to show how tasks are distributed among compute nodes and node cores.
+This console application sample project backs the code snippets found in [Maximize Azure Batch compute resource usage with concurrent node tasks](http://azure.microsoft.com/documentation/articles/batch-parallel-node-tasks/). The application demonstrates the creation of a Batch pool whose compute nodes are configured for executing multiple concurrent tasks, submit tasks with variable slots, and prints node/job/task information to the console during execution to show how tasks are distributed among compute nodes and node cores.
+
+1. Open the project in **Visual Studio 2017**.
+2. Add your Batch and Storage **account credentials** to **accountsettings.json** in the Microsoft.Azure.Batch.Samples.Common project.
+3. **Build** and then **run** the solution. Restore any NuGet packages if prompted.

CSharp/ArticleProjects/PersistOutputs/PersistOutputs.csproj

@@ -17,9 +17,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Batch" Version="12.0.0" />
+    <PackageReference Include="Microsoft.Azure.Batch" Version="14.0.0" />
     <PackageReference Include="Microsoft.Azure.Batch.Conventions.Files" Version="3.5.1" />
-  
+
   </ItemGroup>
 
 </Project>