fix template provisioning by verifying VM is provisioned before doing anything else

Author: com6056

gunicorn_conf.py

@@ -16,12 +16,11 @@
 def start_websockify(websockify_path, target_file):
     result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE)
     if not result.stdout:
-        subprocess.call(
-            [
-                websockify_path, '8081', '--token-plugin', 'TokenFile',
-                '--token-source', target_file, '-D'
-            ],
-            stdout=subprocess.PIPE)
+        subprocess.call([
+            websockify_path, '8081', '--token-plugin', 'TokenFile',
+            '--token-source', target_file, '-D'
+        ],
+                        stdout=subprocess.PIPE)
 
 
 def on_starting(server):

proxstar/__init__.py

@@ -140,8 +140,9 @@ def list_vms(user_view=None):
         if user.active:
             vms = user.vms
             for pending_vm in user.pending_vms:
-                vm = next((vm for vm in vms
-                           if vm['name'] == pending_vm['name']), None)
+                vm = next(
+                    (vm for vm in vms if vm['name'] == pending_vm['name']),
+                    None)
                 if vm:
                     vms[vms.index(vm)]['status'] = pending_vm['status']
                     vms[vms.index(vm)]['pending'] = True

proxstar/tasks.py

@@ -104,8 +104,8 @@ def process_expiring_vms_task():
                         vm.stop()
                 elif days <= -7:
                     print(
-                        "Deleting {} ({}) as it has been at least a week since expiration.".
-                        format(vm.name, vm.id))
+                        "Deleting {} ({}) as it has been at least a week since expiration."
+                        .format(vm.name, vm.id))
                     send_stop_ssh_tunnel(vm.id)
                     delete_vm_task(vm.id)
             if expiring_vms:
@@ -135,15 +135,29 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory):
         job.meta['status'] = 'cloning template'
         job.save_meta()
         vmid, mac = clone_vm(proxmox, template_id, name, user)
+        print("[{}] Waiting until Proxmox is done provisioning.".format(name))
+        job.meta['status'] = 'waiting for Proxmox'
+        job.save_meta()
+        timeout = 200
+        retry = 0
+        while retry < timeout:
+            if not VM(vmid).is_provisioned():
+                retry += 1
+                time.sleep(3)
+                continue
+            break
+        if retry == timeout:
+            print("[{}] Failed to provision, deleting.".format(name))
+            job.meta['status'] = 'failed to provision'
+            job.save_meta()
+            delete_vm_task(vmid)
+            return
         print("[{}] Registering in STARRS.".format(name))
         job.meta['status'] = 'registering in STARRS'
         job.save_meta()
         ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
         register_starrs(starrs, name, app.config['STARRS_USER'], mac, ip)
         get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
-        print("[{}] Giving Proxmox some time to finish cloning.".format(name))
-        job.meta['status'] = 'waiting for Proxmox'
-        time.sleep(15)
         print("[{}] Setting CPU and memory.".format(name))
         job.meta['status'] = 'setting CPU and memory'
         job.save_meta()
@@ -152,12 +166,12 @@ def setup_template_task(template_id, name, user, ssh_key, cores, memory):
         vm.set_mem(memory)
         print("[{}] Applying cloud-init config.".format(name))
         job.meta['status'] = 'applying cloud-init'
+        job.save_meta()
         vm.set_ci_user(user)
         vm.set_ci_ssh_key(ssh_key)
         vm.set_ci_network()
-        print(
-            "[{}] Waiting for STARRS to propogate before starting VM.".format(
-                name))
+        print("[{}] Waiting for STARRS to propogate before starting VM.".
+              format(name))
         job.meta['status'] = 'waiting for STARRS'
         job.save_meta()
         time.sleep(90)

proxstar/vm.py

@@ -37,6 +37,13 @@ def status(self):
     def qmpstatus(self):
         return self.info['qmpstatus']
 
+    def is_provisioned(self):
+        try:
+            self.set_cpu(self.cpu)
+            return True
+        except:
+            return False
+
     @lazy_property
     def node(self):
         proxmox = connect_proxmox()

proxstar/vnc.py

@@ -10,12 +10,11 @@
 def start_websockify(websockify_path, target_file):
     result = subprocess.run(['pgrep', 'websockify'], stdout=subprocess.PIPE)
     if not result.stdout:
-        subprocess.call(
-            [
-                websockify_path, '8081', '--token-plugin', 'TokenFile',
-                '--token-source', target_file, '-D'
-            ],
-            stdout=subprocess.PIPE)
+        subprocess.call([
+            websockify_path, '8081', '--token-plugin', 'TokenFile',
+            '--token-source', target_file, '-D'
+        ],
+                        stdout=subprocess.PIPE)
 
 
 def stop_websockify():
@@ -24,11 +23,11 @@ def stop_websockify():
         pid = result.stdout.strip()
         subprocess.run(['kill', pid], stdout=subprocess.PIPE)
         time.sleep(3)
-        if subprocess.run(
-            ['pgrep', 'websockify'], stdout=subprocess.PIPE).stdout:
+        if subprocess.run(['pgrep', 'websockify'],
+                          stdout=subprocess.PIPE).stdout:
             time.sleep(10)
-            if subprocess.run(
-                ['pgrep', 'websockify'], stdout=subprocess.PIPE).stdout:
+            if subprocess.run(['pgrep', 'websockify'],
+                              stdout=subprocess.PIPE).stdout:
                 print('Websockify didn\'t stop, killing forcefully.')
                 subprocess.run(['kill', '-9', pid], stdout=subprocess.PIPE)
 
@@ -90,8 +89,9 @@ def start_ssh_tunnel(node, port):
 def stop_ssh_tunnel(vmid, ssh_tunnels):
     # Tear down the SSH tunnel and VNC target entry for a given VM
     port = 5900 + int(vmid)
-    tunnel = next((tunnel for tunnel in ssh_tunnels
-                   if tunnel.local_bind_port == port), None)
+    tunnel = next(
+        (tunnel for tunnel in ssh_tunnels if tunnel.local_bind_port == port),
+        None)
     if tunnel:
         print("Tearing down SSH tunnel for VM {}.".format(vmid))
         try: