Merge branch 'main' into hoh-update-vm-connector

Author: olethanh

.dockerignore

@@ -14,3 +14,4 @@
 **/data.tgz
 /pydantic/
 **/target
+/packaging/sevctl/target

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,34 @@
+Explain what problem this PR is resolving
+
+Related ClickUp, GitHub or Jira tickets : ALEPH-XXX
+
+## Self proofreading checklist
+
+- [ ] The new code clear, easy to read and well commented.
+- [ ] New code does not duplicate the functions of builtin or popular libraries.
+- [ ] An LLM was used to review the new code and look for simplifications.
+- [ ] New classes and functions contain docstrings explaining what they provide.
+- [ ] All new code is covered by relevant tests.
+- [ ] Documentation has been updated regarding these changes.
+- [ ] Dependencies update in the project.toml have been mirrored in the Debian package build script `packaging/Makefile`
+
+## Changes
+
+Explain the changes that were made. The idea is not to list exhaustively all the changes made (GitHub already provides a full diff), but to help the reviewers better understand:
+- which specific file changes go together, e.g: when creating a table in the front-end, there usually is a config file that goes with it
+- the reasoning behind some changes, e.g: deleted files because they are now redundant
+- the behaviour to expect, e.g: tooltip has purple background color because the client likes it so, changed a key in the API response to be consistent with other endpoints
+
+## How to test
+
+Explain how to test your PR.
+If a specific config is required explain it here (account, data entry, ...)
+
+## Print screen / video
+
+Upload here screenshots or videos showing the changes if relevant.
+
+## Notes
+
+Things that the reviewers should know: known bugs that are out of the scope of the PR, other trade-offs that were made.
+If the PR depends on a PR in another repo, or merges into another PR (i.o. main), it should also be mentioned here

.github/workflows/build-deb-package.yml

@@ -1,35 +1,49 @@
+---
 name: "Build Packages"
-on:
-  push
+on: push
+
 
 jobs:
   build_deb:
     name: "Build ${{ matrix.os }} Package"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        os: ["debian-11", "debian-12", "ubuntu-22.04"]
+        os: ["debian-12", "ubuntu-22.04", "ubuntu-24.04"]
         include:
-          - os: "debian-11"
-            make_target: "all-podman-debian-11"
-            artifact_name: "aleph-vm.debian-11.deb"
           - os: "debian-12"
             make_target: "all-podman-debian-12"
             artifact_name: "aleph-vm.debian-12.deb"
           - os: "ubuntu-22.04"
             make_target: "all-podman-ubuntu-2204"
             artifact_name: "aleph-vm.ubuntu-22.04.deb"
+          - os: "ubuntu-24.04"
+            make_target: "all-podman-ubuntu-2404"
+            artifact_name: "aleph-vm.ubuntu-24.04.deb"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
+          submodules: true
           # Fetch the whole history for all tags and branches (required for aleph.__version__)
           fetch-depth: 0
 
+      - name: Initialize git submodules
+        run: git submodule init
+
       - run: |
           cd packaging && make ${{ matrix.make_target }} && cd ..
           ls packaging/target
 
+      - name: Ensure that the relevant files are present in the package
+        run: |
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/kubo/ipfs
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/firecracker
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/jailer
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/vmlinux.bin
+          dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/sevctl
+
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}
@@ -40,10 +54,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: ["debian-11", "debian-12"]
+        os: ["debian-12"]
         include:
-          - os: "debian-11"
-            artifact_name: "aleph-debian-11-python.squashfs"
           - os: "debian-12"
             artifact_name: "aleph-debian-12-python.squashfs"
     steps:

.github/workflows/codeql-analysis.yml

@@ -1,3 +1,4 @@
+---
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
@@ -11,15 +12,17 @@
 #
 name: "CodeQL"
 
+
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ main ]
+    branches: [main]
   schedule:
     - cron: '15 16 * * 0'
 
+
 jobs:
   analyze:
     name: Analyze
@@ -32,29 +35,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: ['python']
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,5 +70,5 @@ jobs:
     #   make bootstrap
     #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/deploy-main-on-staging.yml

@@ -0,0 +1,64 @@
+---
+# This workflow automatically deploys main on staging
+name: "Deploy `main` automatically on staging"
+
+
+on:
+  push:
+    branches:
+      - main
+
+
+jobs:
+  deploy_staging_servers:
+    name: "Deploying on ${{ matrix.staging_servers.hostname }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        staging_servers:
+          - hostname: "ovh.staging.aleph.sh"
+            # Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
+            host_keys: "fDF8b3JHVkxyOU83Qnh0QmkvWjd4N0lVTkRDSHFRPXwxZEdZSnNjNlFyejA5QkR6cGROR1BLYjNES009IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHZwSmNpV2dscTNCbEsxY2xOUmNETnVQeFVCeGF3bE5qVElHZFV2MmoyTVo4KzliVVpDSkI1aXFIKzNvbkc5Vklla1RQdW1ybFlXbFMvZkkvSzM3dTh5UXJuQ3JkNi9XRm9XWWJOaTJ4NWxSOUhzaTViRXQ4MFAwRkFyVVpaSkdzbnRQdVhGeFJGR3dHeFNENTN2emVMbmU4VjRlaUxrQ3BjMDU5YzZVVHBublcvSjdRRnlzZURDUXIwVzFsMzBNcjlnTm1LbmpBd2VLWXdCS0hYaG42VGdSd1RYT1E3VXJCc3c2Q1d0OHI2N2g4QkJ2UHQ5OWt5OHl4dUw2Z25TRlhqeWhyKzVhd1lTY3VhVU5JS3B0Y2JFOWpISHhEY1FLSHN0akZZRHRsM0JWN29rUEkvUWJablpSMDVTdDgvZldoK2p5K3ZtR3BTWmtFckJ2NkUwdFhHMDhmdkdheVRXNWFVcWxRQmlLMzJpNmJlUWordjI3b0pUWndvcndBOVJCY1QramlCWVRNVUFpTTJrblhXMGlqT2ViWDNackpITm5QbXJkbjBTd1JldGlLRzg2SGdRK3d3a0dwd3UxVk01eTFlbTVwZ0VUdnU5SHg1RTFKeEJLcXJ3ZkdhTVVRWFZEWG8yNDg5bW1XZzA1aUFHejZIaXNTVWRESFlRKzhnWnA4PQp8MXxvUzkyc1NEb3RxU1hSb0F6MUpFS1V2RDhVTGM9fDVtSHZBSVdqbk1CN2IwcllRQlo0SXBpaFlqQT0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkZNanZFOEFsQmYxbkp1Y0ZlaEJjSUY2RE8wdGJOdU96OEx5QlFUdC82RlEwaWYyWVAxQUJ1TjBmYXVIT3R4WEx6b25vSGVhTDZaV0JoakhmRGV4NlY4PQp8MXxMc2lPc3RhVGk5bEhYSlFsWDJYQ3c3Q0lTU1k9fGk1RzlFTHJydHpaYkUrR2JjbWF1SDIxOG1ZND0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUp1QVNEMWY1d2dXM3pnd3FGalBXYzhPRi9BZ1pmSFFVa3lRMDE2c1MrRmoK"
+            os: "debian-12"
+            make_target: "all-podman-debian-12"
+            artifact_name: "aleph-vm.debian-12.deb"
+
+          - hostname: "hetzner.staging.aleph.sh"
+            # Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
+            host_keys: "fDF8WUlKd0FYWnYxZ24vNkRCU0tkYjg0TC9sUngwPXwrRk96RzdoSTJ5Y3JzUW1uSEwrdEFBQkR4YUU9IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHBKcHF5ajUxWUluRkNyZjZUWjE5eUF3cHlXNTNHaFAxNXQ0Wm56cHBwOUVnNTNnZmVWdmk5WUV1bVV6cnVUN01LdFpobjNsb0U5YVFtRUYzSElpb3c5ZmlCWVA3aUMzUUFGdUJCandPUmQwV1RVWDZQQUN2c2p0b1JLWjJpTWZ2YXdITHdrWHErWnkrc2hHNU44L2pwQlJ4MC9paXJta2xPS0F5QWw0QTYzZ2MxMndsVGQzcS9IcDVxd1dSYVV3M1JVUTFTVVJSN2RGRW81VWxqeUZVYS9zdWV1STBYOXdLd0tPZ09iOEo3ZFZDMEdDT3VibkJXL3Jmb3N0YVV5eStaSzdQdzBrM251M2szTFZuUVlPTGlNOG1NMnJub2ZWZ2RSWXpiM3RTUVVrbk9wektBVzBXK3llWmpSOXp1UG4yNXF4bWxsRmRaNmt3QTFDcWY2MmQyQ0dtQ2NDU3dUSUl4ZHJ3M29oOEZOclpROTI4OGQvcmF4djZXZi9oZDI0Y1JqeDdFSEJxOUFWMW02UTZWeGxnSWl0WjIzODlsYjRmODNGclRrNUtib3J3Zm5oM3NaZFRSSkJqRjRhdHZ5NktsWFYxenROc05BeDhFN1RnTDMzVFlnRGc4RWlldGN1TVlzUlcwSnREdldBNGxsZDFQS3JrbDJ1LzZvLzNUb0xVPQp8MXxmQ3FnTjB2WHpMTnAzdklnZXdkSFRQRTA0ZUk9fDhnSituTC9hUGpEQlRMcUNJak1sZFpVbFRpST0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQktWbnE5aWsvcHZFaDdXbHFydUtWZmdZeTlwOVpNQnVKV2IrZkVvS0hZY0ZSYld5c0lYRjJlalBnaUMyOFEvZExqeUhXd2RVZlMySFBMbGNxRVFEZlpvPQp8MXxtVzA4T3ZqUnh0bmRjYVNyc0poWXBQcXp2akk9fFlDcktMeUg4ZnJJR0lRV05RS3hiUnArNlIvTT0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUl5ZGNhTXF1dkZFTEpNUDBlRmhNUGJWZVBSVjlSUEhVRzhIZGZIQmRvaTEK"
+            os: "debian-12"
+            make_target: "all-podman-debian-12"
+            artifact_name: "aleph-vm.debian-12.deb"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Fetch the whole history for all tags and branches (required for aleph.__version__)
+          fetch-depth: 0
+
+      - run: |
+          cd packaging && make ${{ matrix.staging_servers.make_target }} && cd ..
+          ls packaging/target
+
+      - name: Setup SSH private key
+        run: |
+          mkdir ~/.ssh
+          echo $STAGING_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
+          chmod 0700 ~/.ssh
+          chmod 0600 ~/.ssh/id_ed25519
+        env:
+          # Create using:
+          # ssh-keygen -t ed25519 -f ./id_ed25519
+          # cat ./id_ed25519 | base64 --wrap=0
+          STAGING_SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
+
+      - name: Install Aleph-VM on the Staging servers
+        run: |-
+          echo ${{ matrix.staging_servers.host_keys }} | base64 --decode > ~/.ssh/known_hosts
+
+          # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH.
+          while ssh root@${{ matrix.staging_servers.hostname }} lsof /var/lib/apt/lists/lock; do sleep 1; done
+
+          scp packaging/target/${{ matrix.staging_servers.artifact_name }} root@${{ matrix.staging_servers.hostname }}:/opt
+          ssh root@${{ matrix.staging_servers.hostname }} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y --allow-downgrades /opt/${{ matrix.staging_servers.artifact_name }}"

.github/workflows/pr-rating.yml

@@ -1,12 +1,16 @@
+---
 name: Test PR Difficulty Rating Action
 
+
 permissions:
   pull-requests: write
 
+
 on:
   pull_request:
     types: [opened, reopened, ready_for_review]
 
+
 jobs:
   difficulty-rating:
     runs-on: ubuntu-latest

.github/workflows/test-build-examples.yml

@@ -1,8 +1,8 @@
-
+---
 
 name: "Build Examples"
-on:
-  push
+on: push
+
 
 jobs:
   build_pip:

.github/workflows/test-new-runtime-examples.yml

@@ -1,6 +1,7 @@
+---
 name: "Test new runtime and examples"
-on:
-  push
+on: push
+
 
 jobs:
   run_debian_12:
@@ -74,12 +75,12 @@ jobs:
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
           ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts
-          
+
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y"
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles"
           ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha"
-          
+
           scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt
           scp -pr ./examples root@${DROPLET_IPV4}:/opt/
           ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt -o DPkg::Lock::Timeout=60 install -y /opt/aleph-vm.debian-12.deb"
@@ -91,7 +92,7 @@ jobs:
       - name: Test Aleph-VM on the Droplet
         run: |
           export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)"
-          
+
           sleep 3
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system"
           curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi"
@@ -104,5 +105,5 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: |
+        run: |-
           doctl compute droplet delete -f aleph-vm-ci-runtime