add cancellation support

Author: elasticroentgen

Database/JobState.cs

@@ -6,5 +6,6 @@ public enum JobState
     Queued = 1,
     InProgress = 2,
     Completed = 3,
-    Vanished = 4
+    Vanished = 4,
+    Cancelled = 5
 }

Database/RunnerStatus.cs

@@ -11,5 +11,6 @@ public enum RunnerStatus
     Deleted = 6,
     Failure = 7,
     VanishedOnCloud = 8,
-    Cleanup = 9
+    Cleanup = 9,
+    Cancelled = 10
 }

GithubActionsOrchestrator.IntegrationTests/appsettings.test.json

@@ -2,14 +2,57 @@
   "TestConfiguration": {
     "RunnerPrefix": "ghr-test",
     "MinVmId": 20000,
-    "PveHost": "${PVE_HOST}",
-    "PveUsername": "${PVE_USERNAME}",
-    "PvePassword": "${PVE_PASSWORD}",
+    "PveHost": "10.128.1.2",
+    "PveUsername": "ghrunners@pve",
+    "PvePassword": "sEWqPNArmMFUu6wsCwuXUqMVfAe74F26",
     "PveTemplate": 170,
     "ProvisionScriptBaseUrl": "https://example.com/scripts",
     "MetricUser": "test-user",
     "MetricPassword": "test-password",
     "ControllerUrl": "http://localhost:5000",
-    "GithubAgentVersion": "2.0.0"
+    "GithubAgentVersion": "2.0.0",
+    "DbConnectionString": "Host=localhost;Port=5433;Database=github_actions_orchestrator_test;Username=test_user;Password=test_password",
+    "Sizes": [
+      {
+        "Name": "small",
+        "Arch": "x64",
+        "VmTypes": [
+          {
+            "Cloud": "pve",
+            "VmType": "2c2g",
+            "Priority": 1
+          }
+        ]
+      },
+      {
+        "Name": "medium",
+        "Arch": "x64",
+        "VmTypes": [
+          {
+            "Cloud": "pve",
+            "VmType": "4c4g",
+            "Priority": 1
+          }
+        ]
+      }
+    ],
+    "Profiles": [
+      {
+        "Name": "default",
+        "ScriptName": "ubuntu",
+        "ScriptVersion": 1,
+        "OsImageName": "ubuntu-22.04",
+        "IsCustomImage": false,
+        "UsePrivateNetworks": false
+      },
+      {
+        "Name": "test-profile",
+        "ScriptName": "test",
+        "ScriptVersion": 1,
+        "OsImageName": "test-image",
+        "IsCustomImage": true,
+        "UsePrivateNetworks": false
+      }
+    ]
   }
 }

Models/RunnerQueue.cs

@@ -8,4 +8,6 @@ public class RunnerQueue
     public ConcurrentQueue<DeleteRunnerTask> DeleteTasks { get; } = new();
 
     public ConcurrentDictionary<string, CreateRunnerTask> CreatedRunners { get; } = new();
+
+    public ConcurrentDictionary<(string Owner, string Repository, string Size, string Profile, string Arch), int> CancelledRunners { get; } = new();
 }

PoolManager.cs

@@ -866,7 +866,39 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
     {
         var db = new ActionsRunnerContext();
         var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.RunnerId == rt.RunnerDbId);
-        
+
+        // Check if this runner creation should be skipped due to job cancellation
+        var key = (runner.Owner, rt.RepoName, runner.Size, runner.Profile, runner.Arch);
+        bool shouldSkip = false;
+
+        _queues.CancelledRunners.AddOrUpdate(
+            key,
+            0,  // If key doesn't exist, don't skip
+            (k, count) =>
+            {
+                if (count > 0)
+                {
+                    shouldSkip = true;
+                    return count - 1;  // Decrement counter
+                }
+                return 0;
+            });
+
+        if (shouldSkip)
+        {
+            _logger.LogInformation($"Skipping runner creation for cancelled job: Owner={runner.Owner}, Repo={rt.RepoName}, Size={runner.Size}, Profile={runner.Profile}, Arch={runner.Arch}");
+
+            runner.Lifecycle.Add(new RunnerLifecycle
+            {
+                Status = RunnerStatus.Cancelled,
+                EventTimeUtc = DateTime.UtcNow,
+                Event = "Runner creation skipped - job was cancelled"
+            });
+
+            await db.SaveChangesAsync();
+            return true;
+        }
+
         // Check if cloud is stable atm
 
         var possibleProviders =

Program.cs

@@ -306,11 +306,32 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
                     await JobInProgress(workflowJson, logger, jobId, repoName, orgName);
                     break;
                 case "completed":
+                    string conclusion = String.Empty;
+                    if (json.RootElement.TryGetProperty("conclusion", out JsonElement conclusionJson))
+                    {
+                        conclusion = conclusionJson.GetString() ?? string.Empty;
+                    }
+
                     var dbWorkflowComplete = await db.Jobs.FirstOrDefaultAsync(x => x.GithubJobId == jobId);
-                    dbWorkflowComplete.State = JobState.Completed;
                     dbWorkflowComplete.CompleteTime = DateTime.UtcNow;
-                    await db.SaveChangesAsync();
-                    await JobCompleted(logger, jobId, poolMgr, repoName, orgName, workflowJson);
+                    bool wasCancelled = false;
+                    switch (conclusion)
+                    {
+                        case "cancelled":
+                            dbWorkflowComplete.State = JobState.Cancelled;
+                            await db.SaveChangesAsync();
+                            wasCancelled = true;
+                            
+                            // somehow clean runners from creation queue if not started yet.
+                            
+                            break;
+                        default:
+                            dbWorkflowComplete.State = JobState.Completed;
+                            await db.SaveChangesAsync();
+                            break;
+                    }
+
+                    await JobCompleted(logger, jobId, poolMgr, repoName, orgName, workflowJson, wasCancelled);
                     break;
                 default:
                     logger.LogWarning("Unknown action. Ignoring");
@@ -488,7 +509,7 @@ private static async Task<IResult> AddRunnerManuallyHandler(HttpRequest request,
         return Results.StatusCode(201);
     }
 
-    private static async Task JobCompleted(ILogger<Program> logger, long jobId, RunnerQueue poolMgr, string repoName, string orgName, JsonElement workflowJson)
+    private static async Task JobCompleted(ILogger<Program> logger, long jobId, RunnerQueue poolMgr, string repoName, string orgName, JsonElement workflowJson, bool wasCancelled)
     {
         var db = new ActionsRunnerContext();
         var job = await db.Jobs
@@ -523,7 +544,25 @@ await db.Jobs.AddAsync(new Job
             logger.LogError($"No VM on record for JobID: {jobId}. Trying to re-link to {runnerName}.");
             jobRunner = await db.LinkJobToRunner(jobId, runnerName);
 
-            if (jobRunner == null)
+            if (wasCancelled && jobRunner == null)
+            {
+                // job was cancelled before a runner was picked by the job
+                // it's either still in the creation queue or it is still provisioning
+
+                // Get arch from the requested size
+                string arch = Config.Sizes.FirstOrDefault(x => x.Name == job.RequestedSize)?.Arch ?? "x64";
+
+                // Build the cancellation key
+                var key = (job.Owner, job.Repository, job.RequestedSize, job.RequestedProfile, arch);
+
+                // Increment the cancelled runners counter
+                int count = poolMgr.CancelledRunners.AddOrUpdate(key, 1, (k, currentCount) => currentCount + 1);
+
+                logger.LogInformation($"Registered cancelled job {jobId} for {job.Owner}/{job.Repository} size={job.RequestedSize} profile={job.RequestedProfile} arch={arch}. Counter now at {count}.");
+
+                return;
+            }
+            else if (jobRunner == null)
             {
                 logger.LogError("Unable to link runner. aborting");
                 return;