[Feature:System] Revamp worker setup and fix QEMU (#10530)

### Please check if the PR fulfills these requirements:

* [ ] Tests for the changes have been added/updated (if possible)
* [ ] Documentation has been updated/added if relevant
* [ ] Screenshots are attached to Github PR if visual/UI changes were
made

### What is the current behavior?
<!-- List issue if it fixes/closes/implements one using the "Fixes
#<number>" or "Closes #<number>" syntax -->
Worker configuration currently fails on QEMU due to lack of support for
private networking.

### What is the new behavior?
The QEMU plugin has been modified to add support for socket networking.
The worker management configuration has been revamped to better
accomodate the networking needs of the new QEMU plugin setup.

Testing instructions:
See Submitty/submitty.github.io#621

---------

Co-authored-by: Barb Cutler <[email protected]>

Author: skara9

.pylintrc

@@ -89,6 +89,7 @@ ignore-paths=
     .setup/bin/create_untrusted_users.py,
     .setup/bin/partial_reset.py,
     .setup/bin/update_repo_version.py,
+    .setup/bin/refresh_vagrant_workers.py,
     .setup/bin/reset_system.py,
     .setup/bin/code_watcher.py,
     .setup/bin/setup_sample_emails.py,
@@ -134,7 +135,9 @@ ignore-paths=
     migration/tests/test_handle_migration.py,
     migration/tests/test_dumper.py,
     sample_files/sample_CSV/test_grades.py,
-    sample_files/sample_CSV/verify.py
+    sample_files/sample_CSV/verify.py,
+    vagrant-workers/workers.py,
+    vagrant-workers/generate_workers.py
 
 [FORMAT]
 max-line-length=100

.setup/CONFIGURE_SUBMITTY.py

@@ -554,20 +554,6 @@ def write(x=''):
             }
         }
 
-        vagrant_workers_json = os.path.join(SUBMITTY_REPOSITORY, '.vagrant', 'workers.json')
-        if os.path.isfile(vagrant_workers_json):
-            with open(vagrant_workers_json) as f:
-                vagrant_workers = json.load(f, object_hook=OrderedDict)
- 
-            for worker, data in vagrant_workers.items():
-                worker_dict[worker] = {
-                    "capabilities": capabilities,
-                    "address": data["ip_addr"],
-                    "username": "submitty",
-                    "num_autograding_workers": NUM_GRADING_SCHEDULER_WORKERS,
-                    "enabled": True
-                }
-
         with open(WORKERS_JSON, 'w') as workers_file:
             json.dump(worker_dict, workers_file, indent=4)
 

.setup/bin/refresh_vagrant_workers.py

@@ -0,0 +1,96 @@
+import os
+import shutil
+import pwd
+import json
+import subprocess
+from collections import OrderedDict
+
+SUBMITTY_REPOSITORY = os.environ['SUBMITTY_REPOSITORY']
+SUBMITTY_INSTALL_DIR = os.environ['SUBMITTY_INSTALL_DIR']
+DAEMON_USER = os.environ['DAEMON_USER']
+SUPERVISOR_USER = 'submitty'
+
+vagrant_workers_path = os.path.join(SUBMITTY_REPOSITORY, '.vagrant', 'workers.json')
+autograding_workers_path = os.path.join(SUBMITTY_INSTALL_DIR, 'config', 'autograding_workers.json')
+
+print("Loading existing data...")
+with open(vagrant_workers_path) as file:
+    vagrant_workers_data = json.load(file, object_pairs_hook=OrderedDict)
+
+with open(autograding_workers_path) as file:
+    autograding_workers_data = json.load(file, object_pairs_hook=OrderedDict)
+
+if 'version' in vagrant_workers_data and type(vagrant_workers_data['version']) is int:
+    provider = vagrant_workers_data['provider']
+    vagrant_workers_data = vagrant_workers_data['workers']
+else:
+    print("This script requires a worker configuration of v2 or greater. Please regenerate your configuration with 'vagrant workers generate'.")
+    exit(1)
+print("Done loading data")
+print()
+
+print("Generating SSH credentials...")
+shutil.rmtree("/tmp/worker_keys", True)
+os.makedirs("/tmp/worker_keys", 0o500)
+daemon_stat = pwd.getpwnam(DAEMON_USER)
+os.chown("/tmp/worker_keys", daemon_stat.pw_uid, daemon_stat.pw_gid)
+
+DAEMON_HOME = os.path.realpath(subprocess.check_output(['su', DAEMON_USER, '-c', 'echo $HOME']).strip())
+if not os.path.exists(DAEMON_HOME):
+    print("Error: could not find home directory for daemon user")
+    exit(1)
+
+shutil.rmtree(os.path.join(DAEMON_HOME, b'.ssh'), True)
+subprocess.run(['su', DAEMON_USER, '-c', "ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ''"], check=True)
+print("Done generating")
+print()
+
+ssh_config = ''
+successful_machines = []
+for name, data in vagrant_workers_data.items():
+    print("Attempting to connect to " + name)
+    shutil.copyfile(f"{SUBMITTY_REPOSITORY}/.vagrant/machines/{name}/{provider}/private_key", f"/tmp/worker_keys/{name}")
+    os.chown(f"/tmp/worker_keys/{name}", daemon_stat.pw_uid, daemon_stat.pw_gid)
+    os.chmod(f"/tmp/worker_keys/{name}", 0o400)
+    w = subprocess.run(['su', DAEMON_USER, '-c', f"scp -i /tmp/worker_keys/{name} -o StrictHostKeyChecking=no ~/.ssh/id_rsa.pub root@{data['ip_addr']}:/tmp/workerkey"])
+    w = subprocess.run(['su', DAEMON_USER, '-c', f"ssh -i /tmp/worker_keys/{name} -o StrictHostKeyChecking=no root@{data['ip_addr']} \"chown {SUPERVISOR_USER}:{SUPERVISOR_USER} /tmp/workerkey && su {SUPERVISOR_USER} -c \\\"mkdir -p ~/.ssh && mv /tmp/workerkey ~/.ssh/authorized_keys\\\"\""])
+    if w.returncode == 0:
+        print("Connected to " + name)
+        successful_machines.append(name)
+        ssh_config += f"Host {name}\n  HostName {data['ip_addr']}\n  IdentityFile ~/.ssh/id_rsa\n  User submitty\n"
+    else:
+        print("Failed to connect to " + name)
+
+shutil.rmtree("/tmp/worker_keys", True)
+print()
+
+print("Updating SSH configuration...")
+ssh_config_path = os.path.join(DAEMON_HOME, b'.ssh', b'config')
+with open(ssh_config_path, 'w') as file:
+    file.write(ssh_config)
+os.chown(ssh_config_path, daemon_stat.pw_uid, daemon_stat.pw_gid)
+print("Successfully updated")
+
+print("Writing new autograding configuration...")
+new_autograding_data = OrderedDict()
+new_autograding_data['primary'] = autograding_workers_data['primary']
+
+total = 0
+enabled = 0
+for name, data in vagrant_workers_data.items():
+    worker_data = OrderedDict(autograding_workers_data['primary'])
+    worker_data['username'] = SUPERVISOR_USER
+    worker_data['address'] = data['ip_addr']
+    if name not in successful_machines:
+        worker_data['enabled'] = False
+    else:
+        enabled += 1
+    total += 1
+    new_autograding_data[name] = worker_data
+
+with open(autograding_workers_path, 'w') as file:
+    json.dump(new_autograding_data, file, indent=4)
+print(f"Configuration saved with {enabled}/{total} machines enabled")
+print()
+
+print("DONE")

.setup/install_system.sh

@@ -182,6 +182,7 @@ alias migrator='python3 ${SUBMITTY_REPOSITORY}/migration/run_migrator.py -c ${SU
 alias vagrant_info='cat /etc/motd'
 alias ntp_sync='service ntp stop && ntpd -gq && service ntp start'
 alias recreate_sample_courses='sudo bash /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/bin/recreate_sample_courses.sh'
+alias refresh_vagrant_workers='python3 /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/bin/refresh_vagrant_workers.py'
 systemctl start submitty_autograding_shipper
 systemctl start submitty_autograding_worker
 systemctl start submitty_daemon_jobs_handler
@@ -329,18 +330,6 @@ fi
 
 if ! cut -d ':' -f 1 /etc/passwd | grep -q ${DAEMON_USER} ; then
     useradd -m -c "First Last,RoomNumber,WorkPhone,HomePhone" "${DAEMON_USER}" -s /bin/bash
-    if [ ${WORKER} == 0 ] && [ ${DEV_VM} == 1 ] && [ -f ${SUBMITTY_REPOSITORY}/.vagrant/workers.json ]; then
-        echo -e "attempting to create ssh key for submitty_daemon..."
-        su submitty_daemon -c "cd ~/"
-        su submitty_daemon -c "ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ''"
-        su submitty_daemon -c "echo 'successfully created ssh key'"
-
-        while read -r IP
-        do
-            su submitty_daemon -c "sshpass -p 'submitty' ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no submitty@${IP}"
-        done <<< "$(jq -r ".[].ip_addr" "${SUBMITTY_REPOSITORY}/.vagrant/workers.json")"
-        echo "DONE"
-    fi
 fi
 
 # The VCS directories (/var/local/submitty/vcs) are owned by root:$DAEMONCGI_GROUP

Vagrantfile

@@ -79,29 +79,28 @@ base_boxes[:arm_bento]     = "bento/ubuntu-22.04-arm64"
 base_boxes[:libvirt]       = "generic/ubuntu2204"
 base_boxes[:arm_mac_qemu]  = "perk/ubuntu-2204-arm64"
 
-
-def mount_folders(config, mount_options, type = nil)
+def mount_folders(config, mount_options, type = nil, host = '10.0.2.2')
  # ideally we would use submitty_daemon or something as the owner/group, but since that user doesn't exist
   # till post-provision (and this is mounted before provisioning), we want the group to be 'vagrant'
   # which is guaranteed to exist and that during install_system.sh we add submitty_daemon/submitty_php/etc to the
   # vagrant group so that they can write to this shared folder, primarily just for the log files
   owner = 'root'
   group = 'vagrant'
-  config.vm.synced_folder '.', '/usr/local/submitty/GIT_CHECKOUT/Submitty', create: true, owner: owner, group: group, mount_options: mount_options, smb_host: '10.0.2.2', smb_username: `whoami`.chomp, type: type
+  config.vm.synced_folder '.', '/usr/local/submitty/GIT_CHECKOUT/Submitty', create: true, owner: owner, group: group, mount_options: mount_options, smb_host: host, smb_username: `whoami`.chomp, type: type
 
   optional_repos = %w(AnalysisTools AnalysisToolsTS Lichen RainbowGrades Tutorial CrashCourseCPPSyntax IntroQuantumComputing LichenTestData DockerImages DockerImagesRPI)
   optional_repos.each {|repo|
     repo_path = File.expand_path("../" + repo)
     if File.directory?(repo_path)
-      config.vm.synced_folder repo_path, "/usr/local/submitty/GIT_CHECKOUT/" + repo, owner: owner, group: group, mount_options: mount_options, smb_host: '10.0.2.2', smb_username: `whoami`.chomp, type:type
+      config.vm.synced_folder repo_path, "/usr/local/submitty/GIT_CHECKOUT/" + repo, owner: owner, group: group, mount_options: mount_options, smb_host: host, smb_username: `whoami`.chomp, type: type
     end
   }
 end
 
 def get_workers()
   worker_file = File.join(__dir__, '.vagrant', 'workers.json')
   if File.file?(worker_file)
-    return JSON.parse(File.read(worker_file), symbolize_names: true)
+    return JSON.parse(File.read(worker_file), symbolize_names: true)[:workers]
   else
     return Hash[]
   end
@@ -144,22 +143,44 @@ Vagrant.configure(2) do |config|
   # that one) as well as making sure all non-primary ones have "autostart: false" set
   # so that when we do "vagrant up", it doesn't spin up those machines.
 
-  get_workers.map do |worker_name, data|
-    config.vm.define worker_name do |ubuntu|
-      ubuntu.vm.network 'private_network', ip: data[:ip_addr]
-      ubuntu.vm.network 'forwarded_port', guest: 22, host: data[:ssh_port], id: 'ssh'
-      ubuntu.vm.provision 'shell', inline: gen_script(worker_name, worker: true, base: base_box)
+  if ARGV[0] == 'workers'
+    if apple_silicon
+      exec("arch", "-arm64", "python3", "vagrant-workers/workers.py", *ARGV[1..])
     end
+    exec("python3", "vagrant-workers/workers.py", *ARGV[1..])
   end
 
-  vm_name = 'ubuntu-22.04'
-  config.vm.define vm_name, primary: true do |ubuntu|
-    ubuntu.vm.network 'forwarded_port', guest: 1511, host: ENV.fetch('VM_PORT_SITE', 1511)
-    ubuntu.vm.network 'forwarded_port', guest: 8443, host: ENV.fetch('VM_PORT_WS',   8443)
-    ubuntu.vm.network 'forwarded_port', guest: 5432, host: ENV.fetch('VM_PORT_DB',  16442)
-    ubuntu.vm.network 'forwarded_port', guest: 7000, host: ENV.fetch('VM_PORT_SAML', 7001)
-    ubuntu.vm.network 'forwarded_port', guest:   22, host: ENV.fetch('VM_PORT_SSH',  2222), id: 'ssh'
-    ubuntu.vm.provision 'shell', inline: gen_script(vm_name, base: base_box)
+  if ENV.fetch('WORKER_MODE', '0') == '1'
+    get_workers.map do |worker_name, data|
+      config.vm.define worker_name do |ubuntu|
+        ubuntu.vm.network 'private_network', ip: data[:ip_addr]
+        ubuntu.vm.network 'forwarded_port', guest: 22, host: data[:ssh_port], id: 'ssh' unless data[:ssh_port].nil?
+        ubuntu.vm.provision 'shell', inline: gen_script(worker_name, worker: true, base: true)
+
+        ubuntu.vm.provider "qemu" do |qe, override|
+          qe.ssh_host = data[:ip_addr]
+          qe.ssh_port = 22
+          qe.socket_fd = 3
+          qe.mac_address = data[:mac_addr]
+          mount_folders(override, [], 'smb', ENV.fetch('GATEWAY_IP', '10.0.2.2'))
+        end
+      end
+    end
+  else
+    vm_name = 'ubuntu-22.04'
+    config.vm.define vm_name, primary: true do |ubuntu|
+      ubuntu.vm.network 'forwarded_port', guest: 1511, host: ENV.fetch('VM_PORT_SITE', 1511)
+      ubuntu.vm.network 'forwarded_port', guest: 8443, host: ENV.fetch('VM_PORT_WS',   8443)
+      ubuntu.vm.network 'forwarded_port', guest: 5432, host: ENV.fetch('VM_PORT_DB',  16442)
+      ubuntu.vm.network 'forwarded_port', guest: 7000, host: ENV.fetch('VM_PORT_SAML', 7001)
+      ubuntu.vm.network 'forwarded_port', guest:   22, host: ENV.fetch('VM_PORT_SSH',  2222), id: 'ssh'
+      ubuntu.vm.provision 'shell', inline: gen_script(vm_name, base: base_box)
+
+      ubuntu.vm.provider "qemu" do |qe, override|
+        qe.ssh_port = ENV.fetch('VM_PORT_SSH', 2222)
+        mount_folders(override, [], 'smb')
+      end
+    end
   end
 
   config.vm.provider 'virtualbox' do |vb, override|
@@ -264,10 +285,6 @@ Vagrant.configure(2) do |config|
 
     qe.memory = "2G"
     qe.smp = 2
-
-    qe.ssh_port = ENV.fetch('VM_PORT_SSH', 2222)
-
-    mount_folders(override, [])
   end
 
   config.vm.provision :shell, :inline => " sudo timedatectl set-timezone America/New_York", run: "once"