Ze/labvm status job

CSLabs.Api/Jobs/JobRegistry.cs

@@ -8,9 +8,11 @@ public class JobRegistry : Registry
     {
         public JobRegistry(IServiceProvider provider)
         {
+            // Every minute check the status of each Lab VM within each Lab
+            // Every 3 minutes check to see if we have quorum (at least 50% of the ProxMox nodes are up)
 
             // Schedule new jobs here
-            Schedule(() => new ExampleJob(provider)).ToRunEvery(1).Minutes();
+            Schedule(() => new VmStatusJob(provider)).ToRunEvery(1).Minutes();
 
         }
     }

CSLabs.Api/Jobs/VmStatusJob.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Threading.Tasks;
+using CSLabs.Api.Models;
+using CSLabs.Api.Services;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace CSLabs.Api.Jobs
+{
+    public class VmStatusJob : AsyncJob
+    {
+        private readonly IServiceProvider _provider;
+        public VmStatusJob(IServiceProvider provider)
+        {
+            _provider = provider;
+        }
+
+        protected override async Task ExecuteAsync()
+        {
+            using var scope = _provider.CreateScope();
+            using var context = scope.ServiceProvider.GetService<DefaultContext>();
+
+            // Do job
+            var connectionService = new TestVmConnectionService(context);
+            await connectionService.TestLabVmConnection();
+
+        }
+    }
+}

CSLabs.Api/Models/UserModels/UserLabVm.cs

@@ -36,13 +36,16 @@ public class UserLabVm : Trackable, IQemu
         [InverseProperty(nameof(VmInterfaceInstance.UserLabVm))]
         public List<VmInterfaceInstance> InterfaceInstances { get; set; } = new List<VmInterfaceInstance>();
 
+        public bool Running { get; set; } = true;
+
 
         public static void OnModelCreating(ModelBuilder builder)
         {
             builder.TimeStamps<UserLabVm>();
             builder.Entity<UserLabVm>().HasIndex(u => new {u.UserLabId, u.LabVmId}).IsUnique();
             builder.Entity<UserLabVm>().HasIndex(u => new {u.UserLabId});
             builder.Entity<UserLabVm>().HasIndex(u => new {u.LabVmId});
+            builder.Entity<UserLabVm>().HasIndex(u => new {u.Running});
         }
 
 

CSLabs.Api/Proxmox/ProxmoxDBApi.cs

@@ -0,0 +1,36 @@
+using System.Threading.Tasks;
+using CSLabs.Api.Models;
+using CSLabs.Api.Models.HypervisorModels;
+
+namespace CSLabs.Api.Proxmox
+{
+    public class ProxmoxDBApi : ProxmoxApi
+    {
+        private DefaultContext _context;
+        public ProxmoxDBApi(HypervisorNode hypervisorNode, string password, DefaultContext context) : base(hypervisorNode, password)
+        {
+            _context = context;
+        }
+
+        public new async Task StartVM(int vmId, string targetNode = null)
+        {
+            await base.StartVM(vmId, targetNode);
+            // Save in the database that the VM is started
+            _context.UserLabVms.Find(vmId).Running = true;
+        }
+
+        public new async Task StopVM(int vmId)
+        {
+            await base.StopVM(vmId);
+            // Save in the database that the VM is stopped
+            _context.UserLabVms.Find(vmId).Running = false;
+        }
+
+        public new async Task ShutdownVm(int vmId, int timeout = 20)
+        {
+            await base.ShutdownVm(vmId, timeout);
+            // Save in the database that the VM is stopped
+            _context.UserLabVms.Find(vmId).Running = false;
+        }
+    }
+}

CSLabs.Api/Proxmox/ProxmoxManager.cs

@@ -64,6 +64,12 @@ public ProxmoxApi GetProxmoxApi(HypervisorNode node)
             return new ProxmoxApi(node, password);
         }
 
+        public ProxmoxApi GetProxmoxDBApi(HypervisorNode node)
+        {
+            var password = Cryptography.DecryptString(node.Hypervisor.Password, _encryptionKey);
+            return new ProxmoxDBApi(node, password, _context);
+        }
+
         public ProxmoxApi GetProxmoxApi(UserLab userLab)
         {
             return GetProxmoxApi(userLab.HypervisorNode);

CSLabs.Api/Services/TestVmConnectionService.cs

@@ -0,0 +1,99 @@
+using System.Linq;
+using System.Threading.Tasks;
+using CSLabs.Api.Models;
+using CSLabs.Api.Proxmox;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+
+namespace CSLabs.Api.Services
+{
+    public class TestVmConnectionService
+    {
+        private DefaultContext Context { get; }
+
+        private ProxmoxManager ProxmoxManager;
+
+        public TestVmConnectionService(DefaultContext context)
+        {
+            Context = context;
+        }
+
+        // Recursive helper function
+        public async void AttemptStart(int attempt, int labId, ProxmoxApi api)
+        {
+            try
+            {
+                if (attempt != 3) // first or second attempt
+                {
+                    await api.StartVM(labId);
+                }
+                else // third attempt
+                {
+                    await api.StopVM(labId);
+                    await api.StartVM(labId);
+                }
+
+            }
+            catch (ProxmoxRequestException)
+            {
+                if (attempt != 3)
+                {
+                    AttemptStart(attempt + 1, labId, api);
+                }
+                else
+                {
+                    // TODO
+                    // third attempt at restarting has failed. Something really bad has happened
+                    // and the maintainers need to be emailed
+                }
+            }
+        }
+
+        public async Task<IActionResult> TestLabVmConnection()
+        {
+            var hypervisors = await Context.Hypervisors
+                .Include(h => h.HypervisorNodes)
+                .ToListAsync();
+
+            var labVms = await Context.LabVms
+                .Include(v => v.Id)
+                .ToListAsync();
+
+            var userLabVms = await Context.UserLabVms
+                .Include(v => v.Id)
+                .ToListAsync();
+
+            foreach (var hypervisor in hypervisors) 
+            {
+                var api = ProxmoxManager.GetProxmoxDBApi(hypervisor.HypervisorNodes.First());
+
+                foreach (var labVm in labVms) 
+                { 
+                    try 
+                    {
+                        //actual status of VM
+                        var vmStatus = await api.GetVmStatus(labVm.Id); 
+
+                        //value stored in database
+                        var userLab = userLabVms.Find(x => x.LabVmId.Equals(labVm.Id));
+                        var userLabStatus = userLab.Running;
+
+                        // check if actual status is opposite what is stored in the database
+                        if (vmStatus.IsStopped() != userLabStatus)  
+                        {
+                            AttemptStart(1, labVm.Id, api);
+                        }
+                    }
+                    catch (ProxmoxRequestException) 
+                    {
+                        // something went wrong getting the vm status
+                    }
+
+                }
+            }
+
+            return new OkObjectResult("All LabVMs are up and responding");
+        }
+
+    }
+}